2024-11-12 06:47:22,318 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-12 06:47:22,329 main DEBUG Took 0.009275 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-12 06:47:22,329 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-12 06:47:22,329 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-12 06:47:22,330 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-12 06:47:22,331 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 06:47:22,338 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-12 06:47:22,349 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 06:47:22,350 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 06:47:22,351 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 06:47:22,351 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 06:47:22,351 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 06:47:22,352 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 06:47:22,353 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 06:47:22,353 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 06:47:22,353 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 06:47:22,354 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 06:47:22,354 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 06:47:22,355 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 06:47:22,356 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 06:47:22,356 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 06:47:22,356 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 06:47:22,357 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 06:47:22,357 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 06:47:22,357 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 06:47:22,358 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 06:47:22,358 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 06:47:22,358 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 06:47:22,359 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 06:47:22,359 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 06:47:22,359 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 06:47:22,360 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 06:47:22,360 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-12 06:47:22,362 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 06:47:22,363 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-12 06:47:22,365 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-12 06:47:22,365 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-12 06:47:22,367 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-12 06:47:22,367 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-12 06:47:22,375 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-12 06:47:22,377 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-12 06:47:22,379 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-12 06:47:22,379 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-12 06:47:22,379 main DEBUG createAppenders(={Console}) 2024-11-12 06:47:22,380 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-12 06:47:22,380 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-12 06:47:22,381 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-12 06:47:22,381 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-12 06:47:22,381 main DEBUG OutputStream closed 2024-11-12 06:47:22,382 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-12 06:47:22,382 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-12 06:47:22,382 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-12 06:47:22,453 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-12 06:47:22,456 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-12 06:47:22,457 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-12 06:47:22,458 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-12 06:47:22,458 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-12 06:47:22,459 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-12 06:47:22,459 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-12 06:47:22,459 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-12 06:47:22,460 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-12 06:47:22,460 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-12 06:47:22,460 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-12 06:47:22,461 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-12 06:47:22,461 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-12 06:47:22,461 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-12 06:47:22,462 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-12 06:47:22,462 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-12 06:47:22,462 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-12 06:47:22,463 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-12 06:47:22,465 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12 06:47:22,466 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-12 06:47:22,466 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-12 06:47:22,467 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-12T06:47:22,702 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d 2024-11-12 06:47:22,705 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-12 06:47:22,705 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12T06:47:22,713 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-12T06:47:22,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-12T06:47:22,747 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/cluster_a8cb0a6a-8309-0165-eb73-378e9ec04732, deleteOnExit=true 2024-11-12T06:47:22,748 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-12T06:47:22,749 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/test.cache.data in system properties and HBase conf 2024-11-12T06:47:22,750 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/hadoop.tmp.dir in system properties and HBase conf 2024-11-12T06:47:22,750 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/hadoop.log.dir in system properties and HBase conf 2024-11-12T06:47:22,751 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-12T06:47:22,752 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-12T06:47:22,752 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-12T06:47:22,835 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-12T06:47:22,928 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T06:47:22,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-12T06:47:22,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-12T06:47:22,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-12T06:47:22,935 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T06:47:22,935 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-12T06:47:22,936 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-12T06:47:22,937 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T06:47:22,937 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T06:47:22,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-12T06:47:22,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/nfs.dump.dir in system properties and HBase conf 2024-11-12T06:47:22,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/java.io.tmpdir in system properties and HBase conf 2024-11-12T06:47:22,940 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T06:47:22,940 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-12T06:47:22,941 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-12T06:47:23,938 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-12T06:47:24,008 INFO [Time-limited test {}] log.Log(170): Logging initialized @2409ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-12T06:47:24,082 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T06:47:24,141 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T06:47:24,163 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T06:47:24,164 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T06:47:24,165 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T06:47:24,178 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T06:47:24,181 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/hadoop.log.dir/,AVAILABLE} 2024-11-12T06:47:24,182 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T06:47:24,354 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/java.io.tmpdir/jetty-localhost-33297-hadoop-hdfs-3_4_1-tests_jar-_-any-4247187128690657730/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T06:47:24,365 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:33297} 2024-11-12T06:47:24,366 INFO [Time-limited test {}] server.Server(415): Started @2767ms 2024-11-12T06:47:24,888 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T06:47:24,896 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T06:47:24,897 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T06:47:24,898 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T06:47:24,898 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T06:47:24,899 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/hadoop.log.dir/,AVAILABLE} 2024-11-12T06:47:24,899 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T06:47:24,998 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/java.io.tmpdir/jetty-localhost-45067-hadoop-hdfs-3_4_1-tests_jar-_-any-2882345381603537585/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T06:47:24,999 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:45067} 2024-11-12T06:47:24,999 INFO [Time-limited test {}] server.Server(415): Started @3400ms 2024-11-12T06:47:25,052 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T06:47:25,894 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/cluster_a8cb0a6a-8309-0165-eb73-378e9ec04732/data/data1/current/BP-229716150-172.17.0.2-1731394043510/current, will proceed with Du for space computation calculation, 2024-11-12T06:47:25,894 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/cluster_a8cb0a6a-8309-0165-eb73-378e9ec04732/data/data2/current/BP-229716150-172.17.0.2-1731394043510/current, will proceed with Du for space computation calculation, 2024-11-12T06:47:25,919 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T06:47:25,961 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdb9638a8f450acfa with lease ID 0x90189f009bdca628: Processing first storage report for DS-b99ac672-3114-46eb-bc3f-e1b7fcd47be3 from datanode DatanodeRegistration(127.0.0.1:46017, datanodeUuid=71d4d1fb-3c76-4c83-bad6-78ee8fa1cd38, infoPort=33227, infoSecurePort=0, ipcPort=43417, storageInfo=lv=-57;cid=testClusterID;nsid=2110522687;c=1731394043510) 2024-11-12T06:47:25,962 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdb9638a8f450acfa with lease ID 0x90189f009bdca628: from storage DS-b99ac672-3114-46eb-bc3f-e1b7fcd47be3 node DatanodeRegistration(127.0.0.1:46017, datanodeUuid=71d4d1fb-3c76-4c83-bad6-78ee8fa1cd38, infoPort=33227, infoSecurePort=0, ipcPort=43417, storageInfo=lv=-57;cid=testClusterID;nsid=2110522687;c=1731394043510), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-12T06:47:25,963 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdb9638a8f450acfa with lease ID 0x90189f009bdca628: Processing first storage report for DS-04d9ee7e-7e9d-4687-94e3-25a07db96f3e from datanode DatanodeRegistration(127.0.0.1:46017, datanodeUuid=71d4d1fb-3c76-4c83-bad6-78ee8fa1cd38, infoPort=33227, infoSecurePort=0, ipcPort=43417, storageInfo=lv=-57;cid=testClusterID;nsid=2110522687;c=1731394043510) 2024-11-12T06:47:25,963 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdb9638a8f450acfa with lease ID 0x90189f009bdca628: from storage DS-04d9ee7e-7e9d-4687-94e3-25a07db96f3e node DatanodeRegistration(127.0.0.1:46017, datanodeUuid=71d4d1fb-3c76-4c83-bad6-78ee8fa1cd38, infoPort=33227, infoSecurePort=0, ipcPort=43417, storageInfo=lv=-57;cid=testClusterID;nsid=2110522687;c=1731394043510), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T06:47:25,972 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d 2024-11-12T06:47:26,040 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/cluster_a8cb0a6a-8309-0165-eb73-378e9ec04732/zookeeper_0, clientPort=62074, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/cluster_a8cb0a6a-8309-0165-eb73-378e9ec04732/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/cluster_a8cb0a6a-8309-0165-eb73-378e9ec04732/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T06:47:26,049 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62074 2024-11-12T06:47:26,063 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T06:47:26,067 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T06:47:26,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741825_1001 (size=7) 2024-11-12T06:47:26,688 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e with version=8 2024-11-12T06:47:26,688 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/hbase-staging 2024-11-12T06:47:26,774 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-12T06:47:27,016 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9f0beee3d6b6:0 server-side Connection retries=45 2024-11-12T06:47:27,024 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T06:47:27,025 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T06:47:27,030 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T06:47:27,030 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T06:47:27,030 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T06:47:27,178 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-12T06:47:27,233 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-12T06:47:27,242 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-12T06:47:27,245 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T06:47:27,267 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 29989 (auto-detected) 2024-11-12T06:47:27,268 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-12T06:47:27,284 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43031 2024-11-12T06:47:27,305 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43031 connecting to ZooKeeper ensemble=127.0.0.1:62074 2024-11-12T06:47:27,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:430310x0, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T06:47:27,424 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43031-0x1012f161b0d0000 connected 2024-11-12T06:47:27,512 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T06:47:27,515 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T06:47:27,523 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T06:47:27,527 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e, hbase.cluster.distributed=false 2024-11-12T06:47:27,551 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T06:47:27,556 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43031 2024-11-12T06:47:27,557 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43031 2024-11-12T06:47:27,557 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43031 2024-11-12T06:47:27,558 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43031 2024-11-12T06:47:27,558 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43031 2024-11-12T06:47:27,654 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9f0beee3d6b6:0 server-side Connection retries=45 2024-11-12T06:47:27,655 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T06:47:27,655 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T06:47:27,656 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T06:47:27,656 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T06:47:27,656 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T06:47:27,659 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T06:47:27,662 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T06:47:27,663 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38777 2024-11-12T06:47:27,665 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38777 connecting to ZooKeeper ensemble=127.0.0.1:62074 2024-11-12T06:47:27,667 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T06:47:27,669 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T06:47:27,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387770x0, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T06:47:27,703 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38777-0x1012f161b0d0001 connected 2024-11-12T06:47:27,703 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38777-0x1012f161b0d0001, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T06:47:27,709 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T06:47:27,718 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T06:47:27,721 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38777-0x1012f161b0d0001, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T06:47:27,725 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38777-0x1012f161b0d0001, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T06:47:27,726 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38777 2024-11-12T06:47:27,726 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38777 2024-11-12T06:47:27,727 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38777 2024-11-12T06:47:27,728 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38777 2024-11-12T06:47:27,728 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38777 2024-11-12T06:47:27,742 DEBUG [M:0;9f0beee3d6b6:43031 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9f0beee3d6b6:43031 2024-11-12T06:47:27,743 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9f0beee3d6b6,43031,1731394046864 2024-11-12T06:47:27,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x1012f161b0d0001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T06:47:27,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T06:47:27,761 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9f0beee3d6b6,43031,1731394046864 2024-11-12T06:47:27,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x1012f161b0d0001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T06:47:27,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T06:47:27,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x1012f161b0d0001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T06:47:27,795 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T06:47:27,796 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9f0beee3d6b6,43031,1731394046864 from backup master directory 2024-11-12T06:47:27,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9f0beee3d6b6,43031,1731394046864 2024-11-12T06:47:27,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x1012f161b0d0001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T06:47:27,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T06:47:27,809 WARN [master/9f0beee3d6b6:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T06:47:27,809 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9f0beee3d6b6,43031,1731394046864 2024-11-12T06:47:27,813 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-12T06:47:27,814 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-12T06:47:27,873 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/hbase.id] with ID: 6fe3efea-1c5d-4422-afa1-c9a075a9eabd 2024-11-12T06:47:27,873 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/.tmp/hbase.id 2024-11-12T06:47:27,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741826_1002 (size=42) 2024-11-12T06:47:28,289 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/.tmp/hbase.id]:[hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/hbase.id] 2024-11-12T06:47:28,345 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T06:47:28,349 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-12T06:47:28,368 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-12T06:47:28,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x1012f161b0d0001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T06:47:28,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T06:47:28,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741827_1003 (size=196) 2024-11-12T06:47:28,872 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T06:47:28,875 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T06:47:28,888 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:28,891 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T06:47:28,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741828_1004 (size=1189) 2024-11-12T06:47:29,345 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/data/master/store 2024-11-12T06:47:29,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741829_1005 (size=34) 2024-11-12T06:47:29,778 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-12T06:47:29,781 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T06:47:29,782 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T06:47:29,783 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T06:47:29,783 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T06:47:29,784 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T06:47:29,784 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T06:47:29,785 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T06:47:29,786 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731394049782Disabling compacts and flushes for region at 1731394049782Disabling writes for close at 1731394049784 (+2 ms)Writing region close event to WAL at 1731394049785 (+1 ms)Closed at 1731394049785 2024-11-12T06:47:29,788 WARN [master/9f0beee3d6b6:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/data/master/store/.initializing 2024-11-12T06:47:29,788 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/WALs/9f0beee3d6b6,43031,1731394046864 2024-11-12T06:47:29,797 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T06:47:29,811 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9f0beee3d6b6%2C43031%2C1731394046864, suffix=, logDir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/WALs/9f0beee3d6b6,43031,1731394046864, archiveDir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/oldWALs, maxLogs=10 2024-11-12T06:47:29,833 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/WALs/9f0beee3d6b6,43031,1731394046864/9f0beee3d6b6%2C43031%2C1731394046864.1731394049816, exclude list is [], retry=0 2024-11-12T06:47:29,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46017,DS-b99ac672-3114-46eb-bc3f-e1b7fcd47be3,DISK] 2024-11-12T06:47:29,851 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-12T06:47:29,885 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/WALs/9f0beee3d6b6,43031,1731394046864/9f0beee3d6b6%2C43031%2C1731394046864.1731394049816 2024-11-12T06:47:29,886 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33227:33227)] 2024-11-12T06:47:29,886 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T06:47:29,887 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T06:47:29,889 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T06:47:29,890 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T06:47:29,923 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T06:47:29,946 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T06:47:29,949 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:47:29,951 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T06:47:29,952 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T06:47:29,955 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T06:47:29,956 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:47:29,957 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:47:29,957 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T06:47:29,960 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T06:47:29,960 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:47:29,961 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:47:29,961 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T06:47:29,964 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T06:47:29,964 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:47:29,965 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:47:29,966 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T06:47:29,970 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T06:47:29,971 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T06:47:29,976 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T06:47:29,977 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T06:47:29,981 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T06:47:29,985 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T06:47:29,989 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T06:47:29,991 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58737544, jitterRate=-0.12474238872528076}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T06:47:29,998 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731394049901Initializing all the Stores at 1731394049904 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731394049904Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394049905 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394049905Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394049905Cleaning up temporary data from old regions at 1731394049977 (+72 ms)Region opened successfully at 1731394049998 (+21 ms) 2024-11-12T06:47:29,999 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T06:47:30,032 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7670b920, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9f0beee3d6b6/172.17.0.2:0 2024-11-12T06:47:30,059 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-12T06:47:30,070 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-12T06:47:30,070 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-12T06:47:30,073 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T06:47:30,075 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-12T06:47:30,081 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-12T06:47:30,081 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T06:47:30,107 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T06:47:30,117 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-12T06:47:30,160 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-12T06:47:30,162 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T06:47:30,164 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-12T06:47:30,174 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-12T06:47:30,176 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T06:47:30,181 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-12T06:47:30,191 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-12T06:47:30,193 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-12T06:47:30,201 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-12T06:47:30,218 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T06:47:30,226 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-12T06:47:30,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x1012f161b0d0001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T06:47:30,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T06:47:30,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x1012f161b0d0001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T06:47:30,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T06:47:30,238 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9f0beee3d6b6,43031,1731394046864, sessionid=0x1012f161b0d0000, setting cluster-up flag (Was=false) 2024-11-12T06:47:30,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x1012f161b0d0001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T06:47:30,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T06:47:30,293 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-12T06:47:30,297 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9f0beee3d6b6,43031,1731394046864 2024-11-12T06:47:30,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x1012f161b0d0001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T06:47:30,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T06:47:30,343 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-12T06:47:30,346 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9f0beee3d6b6,43031,1731394046864 2024-11-12T06:47:30,355 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-12T06:47:30,426 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T06:47:30,435 INFO [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(746): ClusterId : 6fe3efea-1c5d-4422-afa1-c9a075a9eabd 2024-11-12T06:47:30,438 DEBUG [RS:0;9f0beee3d6b6:38777 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T06:47:30,438 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-12T06:47:30,446 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T06:47:30,453 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9f0beee3d6b6,43031,1731394046864 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T06:47:30,470 DEBUG [RS:0;9f0beee3d6b6:38777 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T06:47:30,470 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9f0beee3d6b6:0, corePoolSize=5, maxPoolSize=5 2024-11-12T06:47:30,470 DEBUG [RS:0;9f0beee3d6b6:38777 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T06:47:30,471 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9f0beee3d6b6:0, corePoolSize=5, maxPoolSize=5 2024-11-12T06:47:30,471 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9f0beee3d6b6:0, corePoolSize=5, maxPoolSize=5 2024-11-12T06:47:30,471 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9f0beee3d6b6:0, corePoolSize=5, maxPoolSize=5 2024-11-12T06:47:30,471 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9f0beee3d6b6:0, corePoolSize=10, maxPoolSize=10 2024-11-12T06:47:30,471 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9f0beee3d6b6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T06:47:30,472 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9f0beee3d6b6:0, corePoolSize=2, maxPoolSize=2 2024-11-12T06:47:30,472 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9f0beee3d6b6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T06:47:30,474 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731394080474 2024-11-12T06:47:30,476 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T06:47:30,478 DEBUG [RS:0;9f0beee3d6b6:38777 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T06:47:30,478 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T06:47:30,478 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T06:47:30,478 DEBUG [RS:0;9f0beee3d6b6:38777 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bd41370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9f0beee3d6b6/172.17.0.2:0 2024-11-12T06:47:30,478 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-12T06:47:30,482 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T06:47:30,482 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T06:47:30,483 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T06:47:30,483 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T06:47:30,485 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:47:30,485 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T06:47:30,486 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T06:47:30,490 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T06:47:30,492 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T06:47:30,492 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T06:47:30,494 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T06:47:30,495 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T06:47:30,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741831_1007 (size=1321) 2024-11-12T06:47:30,497 DEBUG [RS:0;9f0beee3d6b6:38777 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9f0beee3d6b6:38777 2024-11-12T06:47:30,497 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9f0beee3d6b6:0:becomeActiveMaster-HFileCleaner.large.0-1731394050496,5,FailOnTimeoutGroup] 2024-11-12T06:47:30,498 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-12T06:47:30,499 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e 2024-11-12T06:47:30,501 INFO [RS:0;9f0beee3d6b6:38777 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T06:47:30,501 INFO [RS:0;9f0beee3d6b6:38777 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T06:47:30,501 DEBUG [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T06:47:30,502 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9f0beee3d6b6:0:becomeActiveMaster-HFileCleaner.small.0-1731394050497,5,FailOnTimeoutGroup] 2024-11-12T06:47:30,503 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T06:47:30,503 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T06:47:30,504 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T06:47:30,504 INFO [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(2659): reportForDuty to master=9f0beee3d6b6,43031,1731394046864 with port=38777, startcode=1731394047622 2024-11-12T06:47:30,504 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T06:47:30,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741832_1008 (size=32) 2024-11-12T06:47:30,519 DEBUG [RS:0;9f0beee3d6b6:38777 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T06:47:30,549 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49401, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T06:47:30,556 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43031 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:30,558 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43031 {}] master.ServerManager(517): Registering regionserver=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:30,574 DEBUG [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e 2024-11-12T06:47:30,574 DEBUG [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44429 2024-11-12T06:47:30,574 DEBUG [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T06:47:30,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T06:47:30,600 DEBUG [RS:0;9f0beee3d6b6:38777 {}] zookeeper.ZKUtil(111): regionserver:38777-0x1012f161b0d0001, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:30,601 WARN [RS:0;9f0beee3d6b6:38777 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T06:47:30,601 INFO [RS:0;9f0beee3d6b6:38777 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T06:47:30,602 DEBUG [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/WALs/9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:30,604 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9f0beee3d6b6,38777,1731394047622] 2024-11-12T06:47:30,629 INFO [RS:0;9f0beee3d6b6:38777 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T06:47:30,644 INFO [RS:0;9f0beee3d6b6:38777 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T06:47:30,649 INFO [RS:0;9f0beee3d6b6:38777 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T06:47:30,649 INFO [RS:0;9f0beee3d6b6:38777 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T06:47:30,650 INFO [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T06:47:30,655 INFO [RS:0;9f0beee3d6b6:38777 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T06:47:30,656 INFO [RS:0;9f0beee3d6b6:38777 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T06:47:30,656 DEBUG [RS:0;9f0beee3d6b6:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9f0beee3d6b6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T06:47:30,656 DEBUG [RS:0;9f0beee3d6b6:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9f0beee3d6b6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T06:47:30,656 DEBUG [RS:0;9f0beee3d6b6:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9f0beee3d6b6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T06:47:30,657 DEBUG [RS:0;9f0beee3d6b6:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T06:47:30,657 DEBUG [RS:0;9f0beee3d6b6:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9f0beee3d6b6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T06:47:30,657 DEBUG [RS:0;9f0beee3d6b6:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9f0beee3d6b6:0, corePoolSize=2, maxPoolSize=2 2024-11-12T06:47:30,657 DEBUG [RS:0;9f0beee3d6b6:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9f0beee3d6b6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T06:47:30,657 DEBUG [RS:0;9f0beee3d6b6:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9f0beee3d6b6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T06:47:30,657 DEBUG [RS:0;9f0beee3d6b6:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9f0beee3d6b6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T06:47:30,657 DEBUG [RS:0;9f0beee3d6b6:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9f0beee3d6b6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T06:47:30,658 DEBUG [RS:0;9f0beee3d6b6:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9f0beee3d6b6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T06:47:30,658 DEBUG [RS:0;9f0beee3d6b6:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9f0beee3d6b6:0, corePoolSize=1, maxPoolSize=1 2024-11-12T06:47:30,658 DEBUG [RS:0;9f0beee3d6b6:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9f0beee3d6b6:0, corePoolSize=3, maxPoolSize=3 2024-11-12T06:47:30,658 DEBUG [RS:0;9f0beee3d6b6:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0, corePoolSize=3, maxPoolSize=3 2024-11-12T06:47:30,659 INFO [RS:0;9f0beee3d6b6:38777 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T06:47:30,659 INFO [RS:0;9f0beee3d6b6:38777 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T06:47:30,659 INFO [RS:0;9f0beee3d6b6:38777 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T06:47:30,659 INFO [RS:0;9f0beee3d6b6:38777 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T06:47:30,660 INFO [RS:0;9f0beee3d6b6:38777 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T06:47:30,660 INFO [RS:0;9f0beee3d6b6:38777 {}] hbase.ChoreService(168): Chore ScheduledChore name=9f0beee3d6b6,38777,1731394047622-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T06:47:30,678 INFO [RS:0;9f0beee3d6b6:38777 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T06:47:30,680 INFO [RS:0;9f0beee3d6b6:38777 {}] hbase.ChoreService(168): Chore ScheduledChore name=9f0beee3d6b6,38777,1731394047622-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T06:47:30,680 INFO [RS:0;9f0beee3d6b6:38777 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T06:47:30,681 INFO [RS:0;9f0beee3d6b6:38777 {}] regionserver.Replication(171): 9f0beee3d6b6,38777,1731394047622 started 2024-11-12T06:47:30,698 INFO [RS:0;9f0beee3d6b6:38777 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T06:47:30,698 INFO [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(1482): Serving as 9f0beee3d6b6,38777,1731394047622, RpcServer on 9f0beee3d6b6/172.17.0.2:38777, sessionid=0x1012f161b0d0001 2024-11-12T06:47:30,699 DEBUG [RS:0;9f0beee3d6b6:38777 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T06:47:30,699 DEBUG [RS:0;9f0beee3d6b6:38777 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:30,699 DEBUG [RS:0;9f0beee3d6b6:38777 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9f0beee3d6b6,38777,1731394047622' 2024-11-12T06:47:30,699 DEBUG [RS:0;9f0beee3d6b6:38777 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T06:47:30,701 DEBUG [RS:0;9f0beee3d6b6:38777 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T06:47:30,702 DEBUG [RS:0;9f0beee3d6b6:38777 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T06:47:30,702 DEBUG [RS:0;9f0beee3d6b6:38777 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T06:47:30,702 DEBUG [RS:0;9f0beee3d6b6:38777 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:30,702 DEBUG [RS:0;9f0beee3d6b6:38777 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9f0beee3d6b6,38777,1731394047622' 2024-11-12T06:47:30,702 DEBUG [RS:0;9f0beee3d6b6:38777 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T06:47:30,703 DEBUG [RS:0;9f0beee3d6b6:38777 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T06:47:30,703 DEBUG [RS:0;9f0beee3d6b6:38777 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T06:47:30,703 INFO [RS:0;9f0beee3d6b6:38777 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T06:47:30,704 INFO [RS:0;9f0beee3d6b6:38777 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T06:47:30,811 INFO [RS:0;9f0beee3d6b6:38777 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T06:47:30,814 INFO [RS:0;9f0beee3d6b6:38777 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9f0beee3d6b6%2C38777%2C1731394047622, suffix=, logDir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/WALs/9f0beee3d6b6,38777,1731394047622, archiveDir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/oldWALs, maxLogs=32 2024-11-12T06:47:30,830 DEBUG [RS:0;9f0beee3d6b6:38777 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/WALs/9f0beee3d6b6,38777,1731394047622/9f0beee3d6b6%2C38777%2C1731394047622.1731394050817, exclude list is [], retry=0 2024-11-12T06:47:30,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46017,DS-b99ac672-3114-46eb-bc3f-e1b7fcd47be3,DISK] 2024-11-12T06:47:30,839 INFO [RS:0;9f0beee3d6b6:38777 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/WALs/9f0beee3d6b6,38777,1731394047622/9f0beee3d6b6%2C38777%2C1731394047622.1731394050817 2024-11-12T06:47:30,839 DEBUG [RS:0;9f0beee3d6b6:38777 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33227:33227)] 2024-11-12T06:47:30,914 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T06:47:30,919 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T06:47:30,924 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T06:47:30,924 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:47:30,925 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T06:47:30,925 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T06:47:30,928 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T06:47:30,928 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:47:30,929 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T06:47:30,929 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T06:47:30,932 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T06:47:30,932 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:47:30,933 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T06:47:30,933 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T06:47:30,936 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T06:47:30,936 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:47:30,937 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T06:47:30,937 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T06:47:30,939 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740 2024-11-12T06:47:30,939 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740 2024-11-12T06:47:30,943 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T06:47:30,943 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T06:47:30,944 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T06:47:30,946 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T06:47:30,950 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T06:47:30,951 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58913104, jitterRate=-0.12212634086608887}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T06:47:30,953 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731394050914Initializing all the Stores at 1731394050918 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731394050918Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731394050919 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394050919Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731394050919Cleaning up temporary data from old regions at 1731394050943 (+24 ms)Region opened successfully at 1731394050953 (+10 ms) 2024-11-12T06:47:30,953 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T06:47:30,954 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T06:47:30,954 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T06:47:30,954 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T06:47:30,954 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T06:47:30,955 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T06:47:30,955 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731394050953Disabling compacts and flushes for region at 1731394050953Disabling writes for close at 1731394050954 (+1 ms)Writing region close event to WAL at 1731394050955 (+1 ms)Closed at 1731394050955 2024-11-12T06:47:30,958 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T06:47:30,959 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-12T06:47:30,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T06:47:30,971 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T06:47:30,974 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T06:47:31,130 DEBUG [9f0beee3d6b6:43031 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-12T06:47:31,146 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:31,151 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9f0beee3d6b6,38777,1731394047622, state=OPENING 2024-11-12T06:47:31,209 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T06:47:31,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T06:47:31,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x1012f161b0d0001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T06:47:31,220 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T06:47:31,220 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T06:47:31,222 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T06:47:31,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9f0beee3d6b6,38777,1731394047622}] 2024-11-12T06:47:31,408 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T06:47:31,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39233, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T06:47:31,420 INFO [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T06:47:31,421 INFO [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T06:47:31,421 INFO [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T06:47:31,425 INFO [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9f0beee3d6b6%2C38777%2C1731394047622.meta, suffix=.meta, logDir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/WALs/9f0beee3d6b6,38777,1731394047622, archiveDir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/oldWALs, maxLogs=32 2024-11-12T06:47:31,440 DEBUG [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/WALs/9f0beee3d6b6,38777,1731394047622/9f0beee3d6b6%2C38777%2C1731394047622.meta.1731394051427.meta, exclude list is [], retry=0 2024-11-12T06:47:31,443 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46017,DS-b99ac672-3114-46eb-bc3f-e1b7fcd47be3,DISK] 2024-11-12T06:47:31,447 INFO [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/WALs/9f0beee3d6b6,38777,1731394047622/9f0beee3d6b6%2C38777%2C1731394047622.meta.1731394051427.meta 2024-11-12T06:47:31,447 DEBUG [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33227:33227)] 2024-11-12T06:47:31,447 DEBUG [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T06:47:31,449 DEBUG [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T06:47:31,452 DEBUG [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T06:47:31,456 INFO [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T06:47:31,460 DEBUG [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T06:47:31,461 DEBUG [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T06:47:31,461 DEBUG [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T06:47:31,461 DEBUG [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T06:47:31,465 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T06:47:31,467 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T06:47:31,467 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:47:31,468 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T06:47:31,468 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T06:47:31,470 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T06:47:31,470 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:47:31,471 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T06:47:31,471 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T06:47:31,472 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T06:47:31,472 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:47:31,473 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T06:47:31,473 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T06:47:31,475 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T06:47:31,475 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:47:31,476 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T06:47:31,476 DEBUG [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T06:47:31,479 DEBUG [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740 2024-11-12T06:47:31,482 DEBUG [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740 2024-11-12T06:47:31,485 DEBUG [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T06:47:31,485 DEBUG [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T06:47:31,486 DEBUG [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T06:47:31,489 DEBUG [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T06:47:31,491 INFO [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62056104, jitterRate=-0.07529199123382568}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T06:47:31,491 DEBUG [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T06:47:31,493 DEBUG [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731394051462Writing region info on filesystem at 1731394051462Initializing all the Stores at 1731394051464 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731394051464Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731394051465 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394051465Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731394051465Cleaning up temporary data from old regions at 1731394051485 (+20 ms)Running coprocessor post-open hooks at 1731394051491 (+6 ms)Region opened successfully at 1731394051493 (+2 ms) 2024-11-12T06:47:31,500 INFO [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731394051400 2024-11-12T06:47:31,511 DEBUG [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T06:47:31,512 INFO [RS_OPEN_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T06:47:31,514 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:31,517 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9f0beee3d6b6,38777,1731394047622, state=OPEN 2024-11-12T06:47:31,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x1012f161b0d0001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T06:47:31,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T06:47:31,580 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T06:47:31,580 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T06:47:31,580 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:31,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T06:47:31,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9f0beee3d6b6,38777,1731394047622 in 356 msec 2024-11-12T06:47:31,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T06:47:31,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 628 msec 2024-11-12T06:47:31,600 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T06:47:31,600 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-12T06:47:31,617 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:47:31,619 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:47:31,636 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:47:31,638 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43821, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:47:31,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2720 sec 2024-11-12T06:47:31,658 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731394051658, completionTime=-1 2024-11-12T06:47:31,661 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-12T06:47:31,661 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T06:47:31,685 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-12T06:47:31,685 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731394111685 2024-11-12T06:47:31,685 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731394171685 2024-11-12T06:47:31,685 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-12T06:47:31,688 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9f0beee3d6b6,43031,1731394046864-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T06:47:31,688 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9f0beee3d6b6,43031,1731394046864-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T06:47:31,688 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9f0beee3d6b6,43031,1731394046864-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T06:47:31,690 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9f0beee3d6b6:43031, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T06:47:31,690 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T06:47:31,691 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-12T06:47:31,697 DEBUG [master/9f0beee3d6b6:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-12T06:47:31,718 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.909sec 2024-11-12T06:47:31,719 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T06:47:31,721 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T06:47:31,722 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T06:47:31,722 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T06:47:31,722 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T06:47:31,723 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9f0beee3d6b6,43031,1731394046864-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T06:47:31,724 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9f0beee3d6b6,43031,1731394046864-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T06:47:31,732 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-12T06:47:31,733 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T06:47:31,733 INFO [master/9f0beee3d6b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9f0beee3d6b6,43031,1731394046864-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T06:47:31,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45544f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:31,746 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-12T06:47:31,746 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-12T06:47:31,750 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:47:31,752 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:47:31,759 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:47:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:47:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:47:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c924b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:47:31,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:47:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:31,769 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:47:31,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de67725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:31,773 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:47:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:47:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:47:31,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:47:31,817 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9f0beee3d6b6,43031,1731394046864 2024-11-12T06:47:31,833 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=135, ProcessCount=11, AvailableMemoryMB=7499 2024-11-12T06:47:31,846 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T06:47:31,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9f0beee3d6b6,43031,1731394046864 2024-11-12T06:47:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@509c2579 2024-11-12T06:47:31,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T06:47:31,856 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T06:47:31,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T06:47:31,863 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T06:47:31,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T06:47:31,875 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T06:47:31,877 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-12T06:47:31,877 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:47:31,880 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T06:47:31,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T06:47:31,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741835_1011 (size=1061) 2024-11-12T06:47:31,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T06:47:32,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T06:47:32,300 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e 2024-11-12T06:47:32,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741836_1012 (size=53) 2024-11-12T06:47:32,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T06:47:32,716 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T06:47:32,717 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c0610004ec3b741dc44aa538a4d26ae4, disabling compactions & flushes 2024-11-12T06:47:32,717 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:32,717 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:32,717 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. after waiting 0 ms 2024-11-12T06:47:32,717 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:32,717 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:32,718 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c0610004ec3b741dc44aa538a4d26ae4: Waiting for close lock at 1731394052717Disabling compacts and flushes for region at 1731394052717Disabling writes for close at 1731394052717Writing region close event to WAL at 1731394052717Closed at 1731394052717 2024-11-12T06:47:32,721 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T06:47:32,728 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731394052721"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731394052721"}]},"ts":"1731394052721"} 2024-11-12T06:47:32,734 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T06:47:32,736 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T06:47:32,739 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394052737"}]},"ts":"1731394052737"} 2024-11-12T06:47:32,745 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T06:47:32,748 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0610004ec3b741dc44aa538a4d26ae4, ASSIGN}] 2024-11-12T06:47:32,751 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0610004ec3b741dc44aa538a4d26ae4, ASSIGN 2024-11-12T06:47:32,754 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0610004ec3b741dc44aa538a4d26ae4, ASSIGN; state=OFFLINE, location=9f0beee3d6b6,38777,1731394047622; forceNewPlan=false, retain=false 2024-11-12T06:47:32,906 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c0610004ec3b741dc44aa538a4d26ae4, regionState=OPENING, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:32,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0610004ec3b741dc44aa538a4d26ae4, ASSIGN because future has completed 2024-11-12T06:47:32,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622}] 2024-11-12T06:47:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T06:47:33,078 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:33,078 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} 2024-11-12T06:47:33,078 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:33,079 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T06:47:33,079 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:33,079 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:33,081 INFO [StoreOpener-c0610004ec3b741dc44aa538a4d26ae4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:33,085 INFO [StoreOpener-c0610004ec3b741dc44aa538a4d26ae4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:47:33,085 INFO [StoreOpener-c0610004ec3b741dc44aa538a4d26ae4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0610004ec3b741dc44aa538a4d26ae4 columnFamilyName A 2024-11-12T06:47:33,085 DEBUG [StoreOpener-c0610004ec3b741dc44aa538a4d26ae4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:47:33,086 INFO [StoreOpener-c0610004ec3b741dc44aa538a4d26ae4-1 {}] regionserver.HStore(327): Store=c0610004ec3b741dc44aa538a4d26ae4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:47:33,087 INFO [StoreOpener-c0610004ec3b741dc44aa538a4d26ae4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:33,089 INFO [StoreOpener-c0610004ec3b741dc44aa538a4d26ae4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:47:33,089 INFO [StoreOpener-c0610004ec3b741dc44aa538a4d26ae4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0610004ec3b741dc44aa538a4d26ae4 columnFamilyName B 2024-11-12T06:47:33,089 DEBUG [StoreOpener-c0610004ec3b741dc44aa538a4d26ae4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:47:33,090 INFO [StoreOpener-c0610004ec3b741dc44aa538a4d26ae4-1 {}] regionserver.HStore(327): Store=c0610004ec3b741dc44aa538a4d26ae4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:47:33,090 INFO [StoreOpener-c0610004ec3b741dc44aa538a4d26ae4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:33,092 INFO [StoreOpener-c0610004ec3b741dc44aa538a4d26ae4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:47:33,092 INFO [StoreOpener-c0610004ec3b741dc44aa538a4d26ae4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0610004ec3b741dc44aa538a4d26ae4 columnFamilyName C 2024-11-12T06:47:33,093 DEBUG [StoreOpener-c0610004ec3b741dc44aa538a4d26ae4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:47:33,093 INFO [StoreOpener-c0610004ec3b741dc44aa538a4d26ae4-1 {}] regionserver.HStore(327): Store=c0610004ec3b741dc44aa538a4d26ae4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:47:33,094 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:33,094 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:33,095 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:33,096 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:33,098 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:33,098 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:33,099 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T06:47:33,101 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:33,104 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T06:47:33,105 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c0610004ec3b741dc44aa538a4d26ae4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66347655, jitterRate=-0.01134289801120758}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T06:47:33,105 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:33,106 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c0610004ec3b741dc44aa538a4d26ae4: Running coprocessor pre-open hook at 1731394053079Writing region info on filesystem at 1731394053079Initializing all the Stores at 1731394053081 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394053081Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394053081Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394053081Cleaning up temporary data from old regions at 1731394053098 (+17 ms)Running coprocessor post-open hooks at 1731394053105 (+7 ms)Region opened successfully at 1731394053106 (+1 ms) 2024-11-12T06:47:33,108 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., pid=6, masterSystemTime=1731394053067 2024-11-12T06:47:33,111 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:33,111 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:33,112 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c0610004ec3b741dc44aa538a4d26ae4, regionState=OPEN, openSeqNum=2, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:33,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 because future has completed 2024-11-12T06:47:33,123 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-12T06:47:33,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 in 206 msec 2024-11-12T06:47:33,127 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-12T06:47:33,127 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0610004ec3b741dc44aa538a4d26ae4, ASSIGN in 376 msec 2024-11-12T06:47:33,128 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T06:47:33,129 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394053128"}]},"ts":"1731394053128"} 2024-11-12T06:47:33,132 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T06:47:33,133 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T06:47:33,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2670 sec 2024-11-12T06:47:34,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T06:47:34,031 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:47:34,040 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b2c994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,040 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:47:34,041 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:47:34,042 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:47:34,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:47:34,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:47:34,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a4b96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:47:34,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:47:34,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:34,045 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:47:34,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185f36b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,047 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:47:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:47:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:47:34,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:47:34,054 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T06:47:34,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 9f0beee3d6b6,43031,1731394046864 2024-11-12T06:47:34,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@25ca1f99 2024-11-12T06:47:34,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T06:47:34,059 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T06:47:34,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,065 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:47:34,065 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:47:34,067 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:47:34,068 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:47:34,068 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:47:34,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dd81cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:47:34,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:47:34,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:34,071 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:47:34,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9a6e4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,074 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,074 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:47:34,074 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:47:34,076 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:47:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:47:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:47:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89c0433, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:47:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:47:34,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:34,080 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:47:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ef39c97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,082 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:47:34,083 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:47:34,084 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:47:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:47:34,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:47:34,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8ac8568, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:47:34,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:47:34,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:34,088 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:47:34,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab1aa4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,090 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,090 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:47:34,090 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:47:34,092 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:47:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:47:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:47:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f9668b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:47:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:47:34,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:34,095 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:47:34,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c1e7e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2900b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:47:34,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:47:34,098 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:47:34,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:47:34,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:47:34,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70343832, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:47:34,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:47:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:34,101 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:47:34,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f0717f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,103 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:47:34,103 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:47:34,104 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:47:34,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:47:34,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:47:34,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a9c3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:47:34,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:47:34,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:34,107 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:47:34,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b096a35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,109 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6963d59b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,109 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:47:34,109 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:47:34,110 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:47:34,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:47:34,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:47:34,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a88f812, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:47:34,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:47:34,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:34,112 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:47:34,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c7f9af0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,115 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:47:34,115 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:47:34,116 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:47:34,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:47:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:47:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53def629, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:47:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:47:34,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:34,119 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:47:34,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e3080a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,121 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0b495b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,121 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:47:34,121 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:47:34,122 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:47:34,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:47:34,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:47:34,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2268467d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:47:34,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:47:34,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:34,125 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:47:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0f145b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:34,129 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:47:34,129 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:47:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:47:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:47:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:47:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:47:34,131 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:47:34,133 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:47:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:47:34,134 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43318, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:47:34,134 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:47:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:47:34,134 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:47:34,135 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:47:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:47:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:47:34,137 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43344, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:47:34,137 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:47:34,137 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:47:34,140 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:47:34,141 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:47:34,143 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:47:34,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:47:34,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-12T06:47:34,144 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:47:34,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:47:34,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:47:34,144 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:47:34,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:47:34,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:47:34,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:47:34,146 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:47:34,146 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:47:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:47:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:47:34,147 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:47:34,149 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:47:34,149 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:47:34,151 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:47:34,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T06:47:34,151 DEBUG [Thread-156 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:47:34,152 DEBUG [Thread-156 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:47:34,155 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:47:34,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:47:34,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:47:34,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:47:34,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:47:34,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:47:34,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:47:34,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:47:34,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:47:34,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:34,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0610004ec3b741dc44aa538a4d26ae4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T06:47:34,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=A 2024-11-12T06:47:34,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:34,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=B 2024-11-12T06:47:34,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:34,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=C 2024-11-12T06:47:34,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:34,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T06:47:34,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/b9d0ef4dc5f94173a0ac05b506665759 is 50, key is test_row_0/A:col10/1731394054196/Put/seqid=0 2024-11-12T06:47:34,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T06:47:34,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:34,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:34,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:34,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:34,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:34,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741837_1013 (size=14341) 2024-11-12T06:47:34,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:34,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/b9d0ef4dc5f94173a0ac05b506665759 2024-11-12T06:47:34,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:34,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394114357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:34,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:34,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394114381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:34,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:34,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394114384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:34,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:34,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394114392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:34,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:34,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394114397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:34,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:34,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:34,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:34,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:34,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T06:47:34,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/fe892627c554415283db3b84b5fe1415 is 50, key is test_row_0/B:col10/1731394054196/Put/seqid=0 2024-11-12T06:47:34,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741838_1014 (size=12001) 2024-11-12T06:47:34,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/fe892627c554415283db3b84b5fe1415 2024-11-12T06:47:34,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T06:47:34,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:34,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:34,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:34,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:34,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:34,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:34,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:34,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394114538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:34,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:34,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:34,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394114538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:34,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:34,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:34,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394114539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:34,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:34,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:34,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394114539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:34,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:34,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:34,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394114549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:34,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:34,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/2511efec87894c45a30f8bff4941c220 is 50, key is test_row_0/C:col10/1731394054196/Put/seqid=0 2024-11-12T06:47:34,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741839_1015 (size=12001) 2024-11-12T06:47:34,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/2511efec87894c45a30f8bff4941c220 2024-11-12T06:47:34,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/b9d0ef4dc5f94173a0ac05b506665759 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/b9d0ef4dc5f94173a0ac05b506665759 2024-11-12T06:47:34,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/b9d0ef4dc5f94173a0ac05b506665759, entries=200, sequenceid=13, filesize=14.0 K 2024-11-12T06:47:34,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/fe892627c554415283db3b84b5fe1415 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/fe892627c554415283db3b84b5fe1415 2024-11-12T06:47:34,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/fe892627c554415283db3b84b5fe1415, entries=150, sequenceid=13, filesize=11.7 K 2024-11-12T06:47:34,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/2511efec87894c45a30f8bff4941c220 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/2511efec87894c45a30f8bff4941c220 2024-11-12T06:47:34,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/2511efec87894c45a30f8bff4941c220, entries=150, sequenceid=13, filesize=11.7 K 2024-11-12T06:47:34,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c0610004ec3b741dc44aa538a4d26ae4 in 439ms, sequenceid=13, compaction requested=false 2024-11-12T06:47:34,666 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-12T06:47:34,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:34,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T06:47:34,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:34,682 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing c0610004ec3b741dc44aa538a4d26ae4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T06:47:34,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=A 2024-11-12T06:47:34,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:34,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=B 2024-11-12T06:47:34,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:34,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=C 2024-11-12T06:47:34,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:34,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/0764c294fe0043e4bccde82a3f7aab12 is 50, key is test_row_0/A:col10/1731394054375/Put/seqid=0 2024-11-12T06:47:34,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741840_1016 (size=12001) 2024-11-12T06:47:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:34,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T06:47:34,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394114850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:34,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394114851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:34,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394114868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:34,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394114874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:34,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394114878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:34,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:34,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394114989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:34,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:34,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394114990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:34,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:34,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:34,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394114998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394114999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394114999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,141 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/0764c294fe0043e4bccde82a3f7aab12 2024-11-12T06:47:35,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/fdecbc143aed4323bb577f10b0e6e5a1 is 50, key is test_row_0/B:col10/1731394054375/Put/seqid=0 2024-11-12T06:47:35,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741841_1017 (size=12001) 2024-11-12T06:47:35,190 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/fdecbc143aed4323bb577f10b0e6e5a1 2024-11-12T06:47:35,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394115208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394115209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394115210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394115210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394115211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/1c3a323be5954a9eaee2396bb1747128 is 50, key is test_row_0/C:col10/1731394054375/Put/seqid=0 2024-11-12T06:47:35,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741842_1018 (size=12001) 2024-11-12T06:47:35,242 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/1c3a323be5954a9eaee2396bb1747128 2024-11-12T06:47:35,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/0764c294fe0043e4bccde82a3f7aab12 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/0764c294fe0043e4bccde82a3f7aab12 2024-11-12T06:47:35,282 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/0764c294fe0043e4bccde82a3f7aab12, entries=150, sequenceid=33, filesize=11.7 K 2024-11-12T06:47:35,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/fdecbc143aed4323bb577f10b0e6e5a1 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/fdecbc143aed4323bb577f10b0e6e5a1 2024-11-12T06:47:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T06:47:35,303 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/fdecbc143aed4323bb577f10b0e6e5a1, entries=150, sequenceid=33, filesize=11.7 K 2024-11-12T06:47:35,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/1c3a323be5954a9eaee2396bb1747128 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/1c3a323be5954a9eaee2396bb1747128 2024-11-12T06:47:35,326 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/1c3a323be5954a9eaee2396bb1747128, entries=150, sequenceid=33, filesize=11.7 K 2024-11-12T06:47:35,328 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c0610004ec3b741dc44aa538a4d26ae4 in 645ms, sequenceid=33, compaction requested=false 2024-11-12T06:47:35,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:35,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:35,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-12T06:47:35,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-12T06:47:35,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-12T06:47:35,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1810 sec 2024-11-12T06:47:35,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.2000 sec 2024-11-12T06:47:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:35,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0610004ec3b741dc44aa538a4d26ae4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T06:47:35,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=A 2024-11-12T06:47:35,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:35,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=B 2024-11-12T06:47:35,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:35,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=C 2024-11-12T06:47:35,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:35,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/9806d410050b4270893e4142c22be671 is 50, key is test_row_0/A:col10/1731394055532/Put/seqid=0 2024-11-12T06:47:35,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741843_1019 (size=16681) 2024-11-12T06:47:35,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/9806d410050b4270893e4142c22be671 2024-11-12T06:47:35,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394115581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/8da8c0ec2d8d4431868033152745cad0 is 50, key is test_row_0/B:col10/1731394055532/Put/seqid=0 2024-11-12T06:47:35,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394115582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394115588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394115592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394115592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741844_1020 (size=12001) 2024-11-12T06:47:35,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394115698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394115708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394115708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394115708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394115711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394115908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394115919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394115920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394115920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:35,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394115928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:35,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:35,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:36,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/8da8c0ec2d8d4431868033152745cad0 2024-11-12T06:47:36,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/c612fb00ae65436892c95b95a37af328 is 50, key is test_row_0/C:col10/1731394055532/Put/seqid=0 2024-11-12T06:47:36,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741845_1021 (size=12001) 2024-11-12T06:47:36,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:36,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:36,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394116228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:36,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394116228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:36,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:36,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:36,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394116238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:36,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394116239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:36,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:36,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:36,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:36,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394116239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:36,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:36,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T06:47:36,297 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:47:36,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:47:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-12T06:47:36,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T06:47:36,302 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:47:36,303 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:47:36,303 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:47:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T06:47:36,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T06:47:36,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:36,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:36,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:36,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:36,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:36,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/c612fb00ae65436892c95b95a37af328 2024-11-12T06:47:36,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/9806d410050b4270893e4142c22be671 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/9806d410050b4270893e4142c22be671 2024-11-12T06:47:36,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/9806d410050b4270893e4142c22be671, entries=250, sequenceid=47, filesize=16.3 K 2024-11-12T06:47:36,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/8da8c0ec2d8d4431868033152745cad0 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/8da8c0ec2d8d4431868033152745cad0 2024-11-12T06:47:36,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/8da8c0ec2d8d4431868033152745cad0, entries=150, sequenceid=47, filesize=11.7 K 2024-11-12T06:47:36,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/c612fb00ae65436892c95b95a37af328 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/c612fb00ae65436892c95b95a37af328 2024-11-12T06:47:36,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/c612fb00ae65436892c95b95a37af328, entries=150, sequenceid=47, filesize=11.7 K 2024-11-12T06:47:36,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for c0610004ec3b741dc44aa538a4d26ae4 in 1016ms, sequenceid=47, compaction requested=true 2024-11-12T06:47:36,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:36,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:47:36,553 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:47:36,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:36,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:47:36,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:47:36,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:47:36,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T06:47:36,554 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:47:36,557 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:47:36,557 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:47:36,559 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): c0610004ec3b741dc44aa538a4d26ae4/B is initiating minor compaction (all files) 2024-11-12T06:47:36,559 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0610004ec3b741dc44aa538a4d26ae4/B in TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:36,560 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/fe892627c554415283db3b84b5fe1415, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/fdecbc143aed4323bb577f10b0e6e5a1, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/8da8c0ec2d8d4431868033152745cad0] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp, totalSize=35.2 K 2024-11-12T06:47:36,561 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): c0610004ec3b741dc44aa538a4d26ae4/A is initiating minor compaction (all files) 2024-11-12T06:47:36,561 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0610004ec3b741dc44aa538a4d26ae4/A in TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:36,561 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/b9d0ef4dc5f94173a0ac05b506665759, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/0764c294fe0043e4bccde82a3f7aab12, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/9806d410050b4270893e4142c22be671] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp, totalSize=42.0 K 2024-11-12T06:47:36,562 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting fe892627c554415283db3b84b5fe1415, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731394054196 2024-11-12T06:47:36,562 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9d0ef4dc5f94173a0ac05b506665759, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731394054196 2024-11-12T06:47:36,563 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0764c294fe0043e4bccde82a3f7aab12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731394054343 2024-11-12T06:47:36,563 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting fdecbc143aed4323bb577f10b0e6e5a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731394054343 2024-11-12T06:47:36,564 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9806d410050b4270893e4142c22be671, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731394054862 2024-11-12T06:47:36,566 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 8da8c0ec2d8d4431868033152745cad0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731394054862 2024-11-12T06:47:36,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T06:47:36,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:36,612 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing c0610004ec3b741dc44aa538a4d26ae4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T06:47:36,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=A 2024-11-12T06:47:36,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:36,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=B 2024-11-12T06:47:36,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:36,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=C 2024-11-12T06:47:36,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T06:47:36,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/89cd9e0803c24edcb049ca51532bc467 is 50, key is test_row_0/A:col10/1731394055576/Put/seqid=0 2024-11-12T06:47:36,634 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0610004ec3b741dc44aa538a4d26ae4#A#compaction#10 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:47:36,635 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/fbe03c95159248e28c47d37e72dd6b76 is 50, key is test_row_0/A:col10/1731394055532/Put/seqid=0 2024-11-12T06:47:36,636 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0610004ec3b741dc44aa538a4d26ae4#B#compaction#9 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:47:36,638 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/74458a834c6d46a5892ae82385cc3b0c is 50, key is test_row_0/B:col10/1731394055532/Put/seqid=0 2024-11-12T06:47:36,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741846_1022 (size=12001) 2024-11-12T06:47:36,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741847_1023 (size=12104) 2024-11-12T06:47:36,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741848_1024 (size=12104) 2024-11-12T06:47:36,670 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/fbe03c95159248e28c47d37e72dd6b76 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/fbe03c95159248e28c47d37e72dd6b76 2024-11-12T06:47:36,699 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0610004ec3b741dc44aa538a4d26ae4/A of c0610004ec3b741dc44aa538a4d26ae4 into fbe03c95159248e28c47d37e72dd6b76(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:47:36,700 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:36,705 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., storeName=c0610004ec3b741dc44aa538a4d26ae4/A, priority=13, startTime=1731394056550; duration=0sec 2024-11-12T06:47:36,706 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:47:36,706 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:A 2024-11-12T06:47:36,707 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:47:36,710 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:47:36,710 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): c0610004ec3b741dc44aa538a4d26ae4/C is initiating minor compaction (all files) 2024-11-12T06:47:36,710 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0610004ec3b741dc44aa538a4d26ae4/C in TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:36,710 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/2511efec87894c45a30f8bff4941c220, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/1c3a323be5954a9eaee2396bb1747128, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/c612fb00ae65436892c95b95a37af328] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp, totalSize=35.2 K 2024-11-12T06:47:36,711 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2511efec87894c45a30f8bff4941c220, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731394054196 2024-11-12T06:47:36,713 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c3a323be5954a9eaee2396bb1747128, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731394054343 2024-11-12T06:47:36,714 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting c612fb00ae65436892c95b95a37af328, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731394054862 2024-11-12T06:47:36,750 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0610004ec3b741dc44aa538a4d26ae4#C#compaction#12 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:47:36,751 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/036916536e1d45ffbd2713d04c02b792 is 50, key is test_row_0/C:col10/1731394055532/Put/seqid=0 2024-11-12T06:47:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:36,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:36,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741849_1025 (size=12104) 2024-11-12T06:47:36,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:36,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394116771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394116767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:36,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:36,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394116774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:36,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:36,785 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/036916536e1d45ffbd2713d04c02b792 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/036916536e1d45ffbd2713d04c02b792 2024-11-12T06:47:36,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394116781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:36,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:36,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394116782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:36,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:36,801 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0610004ec3b741dc44aa538a4d26ae4/C of c0610004ec3b741dc44aa538a4d26ae4 into 036916536e1d45ffbd2713d04c02b792(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:47:36,801 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:36,801 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., storeName=c0610004ec3b741dc44aa538a4d26ae4/C, priority=13, startTime=1731394056553; duration=0sec 2024-11-12T06:47:36,802 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:36,802 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:C 2024-11-12T06:47:36,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:36,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394116888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:36,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:36,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:36,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394116888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:36,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:36,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:36,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394116888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:36,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:36,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:36,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394116898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:36,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:36,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:36,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394116898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:36,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:36,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:36,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T06:47:36,974 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T06:47:37,045 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/89cd9e0803c24edcb049ca51532bc467 2024-11-12T06:47:37,045 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-12T06:47:37,071 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/74458a834c6d46a5892ae82385cc3b0c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/74458a834c6d46a5892ae82385cc3b0c 2024-11-12T06:47:37,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/f2f5fd3d30e2407aafaac5e1ae606399 is 50, key is test_row_0/B:col10/1731394055576/Put/seqid=0 2024-11-12T06:47:37,085 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0610004ec3b741dc44aa538a4d26ae4/B of c0610004ec3b741dc44aa538a4d26ae4 into 74458a834c6d46a5892ae82385cc3b0c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:47:37,085 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:37,085 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., storeName=c0610004ec3b741dc44aa538a4d26ae4/B, priority=13, startTime=1731394056553; duration=0sec 2024-11-12T06:47:37,085 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:37,086 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:B 2024-11-12T06:47:37,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741850_1026 (size=12001) 2024-11-12T06:47:37,102 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/f2f5fd3d30e2407aafaac5e1ae606399 2024-11-12T06:47:37,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394117098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:37,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:37,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:37,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394117098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394117098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:37,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:37,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394117108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:37,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:37,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394117108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:37,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:37,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/3355ba47e6824dccb92f98b4e6a32fea is 50, key is test_row_0/C:col10/1731394055576/Put/seqid=0 2024-11-12T06:47:37,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741851_1027 (size=12001) 2024-11-12T06:47:37,144 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/3355ba47e6824dccb92f98b4e6a32fea 2024-11-12T06:47:37,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/89cd9e0803c24edcb049ca51532bc467 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/89cd9e0803c24edcb049ca51532bc467 2024-11-12T06:47:37,176 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/89cd9e0803c24edcb049ca51532bc467, entries=150, sequenceid=66, filesize=11.7 K 2024-11-12T06:47:37,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/f2f5fd3d30e2407aafaac5e1ae606399 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/f2f5fd3d30e2407aafaac5e1ae606399 2024-11-12T06:47:37,200 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/f2f5fd3d30e2407aafaac5e1ae606399, entries=150, sequenceid=66, filesize=11.7 K 2024-11-12T06:47:37,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/3355ba47e6824dccb92f98b4e6a32fea as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/3355ba47e6824dccb92f98b4e6a32fea 2024-11-12T06:47:37,212 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/3355ba47e6824dccb92f98b4e6a32fea, entries=150, sequenceid=66, filesize=11.7 K 2024-11-12T06:47:37,214 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c0610004ec3b741dc44aa538a4d26ae4 in 602ms, sequenceid=66, compaction requested=false 2024-11-12T06:47:37,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:37,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:37,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-12T06:47:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-12T06:47:37,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-12T06:47:37,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 916 msec 2024-11-12T06:47:37,229 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-12T06:47:37,230 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-12T06:47:37,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 928 msec 2024-11-12T06:47:37,234 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T06:47:37,234 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-12T06:47:37,235 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-12T06:47:37,235 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-12T06:47:37,236 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T06:47:37,236 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-12T06:47:37,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0610004ec3b741dc44aa538a4d26ae4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T06:47:37,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:37,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=A 2024-11-12T06:47:37,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:37,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=B 2024-11-12T06:47:37,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:37,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=C 2024-11-12T06:47:37,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:37,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/3befc5db9ca94729bc8c0c9704b4d1de is 50, key is test_row_0/A:col10/1731394056770/Put/seqid=0 2024-11-12T06:47:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T06:47:37,438 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:47:37,438 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:47:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-12T06:47:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T06:47:37,442 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:47:37,443 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:47:37,444 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:47:37,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741852_1028 (size=12001) 2024-11-12T06:47:37,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/3befc5db9ca94729bc8c0c9704b4d1de 2024-11-12T06:47:37,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/ea9bd9bf0fa64ee4a6bb04cf7f018b9e is 50, key is test_row_0/B:col10/1731394056770/Put/seqid=0 2024-11-12T06:47:37,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741853_1029 (size=12001) 2024-11-12T06:47:37,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394117497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:37,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:37,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:37,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394117501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394117504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394117507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:37,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:37,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394117507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:37,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T06:47:37,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-12T06:47:37,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:37,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:37,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:37,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:37,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:37,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:37,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:37,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394117617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:37,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394117617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:37,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394117617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:37,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:37,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:37,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:37,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394117618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:37,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394117618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:37,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-12T06:47:37,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:37,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:37,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:37,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:37,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:37,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T06:47:37,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:37,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394117827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394117827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:37,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:37,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394117827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:37,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:37,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394117837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:37,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:37,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394117838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:37,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:37,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/ea9bd9bf0fa64ee4a6bb04cf7f018b9e 2024-11-12T06:47:37,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-12T06:47:37,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:37,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:37,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:37,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:37,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:37,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/e3484f0bc53141208b0e76829d07a334 is 50, key is test_row_0/C:col10/1731394056770/Put/seqid=0 2024-11-12T06:47:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:37,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741854_1030 (size=12001) 2024-11-12T06:47:37,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/e3484f0bc53141208b0e76829d07a334 2024-11-12T06:47:37,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/3befc5db9ca94729bc8c0c9704b4d1de as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/3befc5db9ca94729bc8c0c9704b4d1de 2024-11-12T06:47:37,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/3befc5db9ca94729bc8c0c9704b4d1de, entries=150, sequenceid=85, filesize=11.7 K 2024-11-12T06:47:37,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/ea9bd9bf0fa64ee4a6bb04cf7f018b9e as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/ea9bd9bf0fa64ee4a6bb04cf7f018b9e 2024-11-12T06:47:37,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/ea9bd9bf0fa64ee4a6bb04cf7f018b9e, entries=150, sequenceid=85, filesize=11.7 K 2024-11-12T06:47:37,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/e3484f0bc53141208b0e76829d07a334 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/e3484f0bc53141208b0e76829d07a334 2024-11-12T06:47:37,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/e3484f0bc53141208b0e76829d07a334, entries=150, sequenceid=85, filesize=11.7 K 2024-11-12T06:47:37,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for c0610004ec3b741dc44aa538a4d26ae4 in 573ms, sequenceid=85, compaction requested=true 2024-11-12T06:47:37,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:37,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:47:37,996 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:47:37,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:37,996 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:47:37,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:47:37,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:37,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:47:37,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:47:37,998 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:47:37,998 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): c0610004ec3b741dc44aa538a4d26ae4/A is initiating minor compaction (all files) 2024-11-12T06:47:37,998 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0610004ec3b741dc44aa538a4d26ae4/A in TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:37,999 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/fbe03c95159248e28c47d37e72dd6b76, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/89cd9e0803c24edcb049ca51532bc467, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/3befc5db9ca94729bc8c0c9704b4d1de] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp, totalSize=35.3 K 2024-11-12T06:47:37,999 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:47:37,999 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): c0610004ec3b741dc44aa538a4d26ae4/B is initiating minor compaction (all files) 2024-11-12T06:47:37,999 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0610004ec3b741dc44aa538a4d26ae4/B in TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:37,999 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/74458a834c6d46a5892ae82385cc3b0c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/f2f5fd3d30e2407aafaac5e1ae606399, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/ea9bd9bf0fa64ee4a6bb04cf7f018b9e] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp, totalSize=35.3 K 2024-11-12T06:47:38,000 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbe03c95159248e28c47d37e72dd6b76, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731394054862 2024-11-12T06:47:38,000 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 74458a834c6d46a5892ae82385cc3b0c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731394054862 2024-11-12T06:47:38,001 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89cd9e0803c24edcb049ca51532bc467, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731394055576 2024-11-12T06:47:38,001 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting f2f5fd3d30e2407aafaac5e1ae606399, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731394055576 2024-11-12T06:47:38,001 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3befc5db9ca94729bc8c0c9704b4d1de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731394056770 2024-11-12T06:47:38,002 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting ea9bd9bf0fa64ee4a6bb04cf7f018b9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731394056770 2024-11-12T06:47:38,023 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0610004ec3b741dc44aa538a4d26ae4#B#compaction#18 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:47:38,024 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/9af5abf59b704ae5b0f080b92f5b2bf5 is 50, key is test_row_0/B:col10/1731394056770/Put/seqid=0 2024-11-12T06:47:38,034 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0610004ec3b741dc44aa538a4d26ae4#A#compaction#19 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:47:38,035 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/67ed5cee0f534a88bbdd44d7136c0ddd is 50, key is test_row_0/A:col10/1731394056770/Put/seqid=0 2024-11-12T06:47:38,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741855_1031 (size=12207) 2024-11-12T06:47:38,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741856_1032 (size=12207) 2024-11-12T06:47:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T06:47:38,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-12T06:47:38,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:38,069 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing c0610004ec3b741dc44aa538a4d26ae4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T06:47:38,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=A 2024-11-12T06:47:38,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:38,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=B 2024-11-12T06:47:38,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:38,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=C 2024-11-12T06:47:38,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:38,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/30a8ad65d21a412385164d71c7fa7949 is 50, key is test_row_0/A:col10/1731394057503/Put/seqid=0 2024-11-12T06:47:38,088 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/67ed5cee0f534a88bbdd44d7136c0ddd as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/67ed5cee0f534a88bbdd44d7136c0ddd 2024-11-12T06:47:38,101 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0610004ec3b741dc44aa538a4d26ae4/A of c0610004ec3b741dc44aa538a4d26ae4 into 67ed5cee0f534a88bbdd44d7136c0ddd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:47:38,102 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:38,102 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., storeName=c0610004ec3b741dc44aa538a4d26ae4/A, priority=13, startTime=1731394057996; duration=0sec 2024-11-12T06:47:38,102 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:47:38,102 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:A 2024-11-12T06:47:38,102 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:47:38,104 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:47:38,104 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): c0610004ec3b741dc44aa538a4d26ae4/C is initiating minor compaction (all files) 2024-11-12T06:47:38,105 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0610004ec3b741dc44aa538a4d26ae4/C in TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:38,105 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/036916536e1d45ffbd2713d04c02b792, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/3355ba47e6824dccb92f98b4e6a32fea, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/e3484f0bc53141208b0e76829d07a334] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp, totalSize=35.3 K 2024-11-12T06:47:38,106 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 036916536e1d45ffbd2713d04c02b792, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731394054862 2024-11-12T06:47:38,107 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3355ba47e6824dccb92f98b4e6a32fea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731394055576 2024-11-12T06:47:38,108 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3484f0bc53141208b0e76829d07a334, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731394056770 2024-11-12T06:47:38,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741857_1033 (size=12001) 2024-11-12T06:47:38,127 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/30a8ad65d21a412385164d71c7fa7949 2024-11-12T06:47:38,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:38,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:38,154 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0610004ec3b741dc44aa538a4d26ae4#C#compaction#21 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:47:38,155 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/42a06668792c495294f92006a2d5d8c3 is 50, key is test_row_0/C:col10/1731394056770/Put/seqid=0 2024-11-12T06:47:38,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/00a381ca78ba42248d3b21cffd947c3c is 50, key is test_row_0/B:col10/1731394057503/Put/seqid=0 2024-11-12T06:47:38,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394118187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394118187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394118187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394118192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394118195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741858_1034 (size=12207) 2024-11-12T06:47:38,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741859_1035 (size=12001) 2024-11-12T06:47:38,222 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/42a06668792c495294f92006a2d5d8c3 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/42a06668792c495294f92006a2d5d8c3 2024-11-12T06:47:38,222 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/00a381ca78ba42248d3b21cffd947c3c 2024-11-12T06:47:38,240 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0610004ec3b741dc44aa538a4d26ae4/C of c0610004ec3b741dc44aa538a4d26ae4 into 42a06668792c495294f92006a2d5d8c3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:47:38,241 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:38,241 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., storeName=c0610004ec3b741dc44aa538a4d26ae4/C, priority=13, startTime=1731394057996; duration=0sec 2024-11-12T06:47:38,241 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:38,241 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:C 2024-11-12T06:47:38,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/8972c365046b447cbbcf9cca9830f047 is 50, key is test_row_0/C:col10/1731394057503/Put/seqid=0 2024-11-12T06:47:38,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741860_1036 (size=12001) 2024-11-12T06:47:38,266 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/8972c365046b447cbbcf9cca9830f047 2024-11-12T06:47:38,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/30a8ad65d21a412385164d71c7fa7949 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/30a8ad65d21a412385164d71c7fa7949 2024-11-12T06:47:38,295 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/30a8ad65d21a412385164d71c7fa7949, entries=150, sequenceid=102, filesize=11.7 K 2024-11-12T06:47:38,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/00a381ca78ba42248d3b21cffd947c3c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/00a381ca78ba42248d3b21cffd947c3c 2024-11-12T06:47:38,307 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/00a381ca78ba42248d3b21cffd947c3c, entries=150, sequenceid=102, filesize=11.7 K 2024-11-12T06:47:38,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394118307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394118308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394118309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394118309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394118309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/8972c365046b447cbbcf9cca9830f047 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/8972c365046b447cbbcf9cca9830f047 2024-11-12T06:47:38,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,334 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/8972c365046b447cbbcf9cca9830f047, entries=150, sequenceid=102, filesize=11.7 K 2024-11-12T06:47:38,337 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c0610004ec3b741dc44aa538a4d26ae4 in 268ms, sequenceid=102, compaction requested=false 2024-11-12T06:47:38,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:38,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:38,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-12T06:47:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-12T06:47:38,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-12T06:47:38,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 897 msec 2024-11-12T06:47:38,350 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 910 msec 2024-11-12T06:47:38,463 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/9af5abf59b704ae5b0f080b92f5b2bf5 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/9af5abf59b704ae5b0f080b92f5b2bf5 2024-11-12T06:47:38,475 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0610004ec3b741dc44aa538a4d26ae4/B of c0610004ec3b741dc44aa538a4d26ae4 into 9af5abf59b704ae5b0f080b92f5b2bf5(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:47:38,475 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:38,475 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., storeName=c0610004ec3b741dc44aa538a4d26ae4/B, priority=13, startTime=1731394057996; duration=0sec 2024-11-12T06:47:38,476 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:38,476 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:B 2024-11-12T06:47:38,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:38,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0610004ec3b741dc44aa538a4d26ae4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T06:47:38,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=A 2024-11-12T06:47:38,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:38,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=B 2024-11-12T06:47:38,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:38,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=C 2024-11-12T06:47:38,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:38,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/386a11924be94287ae0be47c768a7789 is 50, key is test_row_0/A:col10/1731394058518/Put/seqid=0 2024-11-12T06:47:38,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741861_1037 (size=12001) 2024-11-12T06:47:38,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/386a11924be94287ae0be47c768a7789 2024-11-12T06:47:38,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394118543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394118544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394118547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394118551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394118552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/144146f8fd2546a2859fe35bc0608c8a is 50, key is test_row_0/B:col10/1731394058518/Put/seqid=0 2024-11-12T06:47:38,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T06:47:38,578 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:47:38,579 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:47:38,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-12T06:47:38,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T06:47:38,582 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:47:38,584 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:47:38,584 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:47:38,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741862_1038 (size=12001) 2024-11-12T06:47:38,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394118657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394118657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394118667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394118667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394118667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T06:47:38,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T06:47:38,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:38,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:38,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:38,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:38,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:38,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394118867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394118868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394118877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394118877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394118877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:38,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T06:47:38,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:38,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:38,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:38,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:38,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T06:47:38,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/144146f8fd2546a2859fe35bc0608c8a 2024-11-12T06:47:39,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/1037b62e62a24375ae0682e2dbdc33fd is 50, key is test_row_0/C:col10/1731394058518/Put/seqid=0 2024-11-12T06:47:39,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741863_1039 (size=12001) 2024-11-12T06:47:39,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/1037b62e62a24375ae0682e2dbdc33fd 2024-11-12T06:47:39,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/386a11924be94287ae0be47c768a7789 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/386a11924be94287ae0be47c768a7789 2024-11-12T06:47:39,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/386a11924be94287ae0be47c768a7789, entries=150, sequenceid=122, filesize=11.7 K 2024-11-12T06:47:39,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/144146f8fd2546a2859fe35bc0608c8a as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/144146f8fd2546a2859fe35bc0608c8a 2024-11-12T06:47:39,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T06:47:39,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:39,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:39,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:39,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:39,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:39,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:39,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/144146f8fd2546a2859fe35bc0608c8a, entries=150, sequenceid=122, filesize=11.7 K 2024-11-12T06:47:39,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/1037b62e62a24375ae0682e2dbdc33fd as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/1037b62e62a24375ae0682e2dbdc33fd 2024-11-12T06:47:39,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/1037b62e62a24375ae0682e2dbdc33fd, entries=150, sequenceid=122, filesize=11.7 K 2024-11-12T06:47:39,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for c0610004ec3b741dc44aa538a4d26ae4 in 554ms, sequenceid=122, compaction requested=true 2024-11-12T06:47:39,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:39,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:47:39,075 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:47:39,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:39,075 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:47:39,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:47:39,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:39,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:47:39,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:47:39,077 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:47:39,077 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): c0610004ec3b741dc44aa538a4d26ae4/B is initiating minor compaction (all files) 2024-11-12T06:47:39,077 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0610004ec3b741dc44aa538a4d26ae4/B in TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:39,077 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/9af5abf59b704ae5b0f080b92f5b2bf5, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/00a381ca78ba42248d3b21cffd947c3c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/144146f8fd2546a2859fe35bc0608c8a] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp, totalSize=35.4 K 2024-11-12T06:47:39,077 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:47:39,078 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): c0610004ec3b741dc44aa538a4d26ae4/A is initiating minor compaction (all files) 2024-11-12T06:47:39,078 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0610004ec3b741dc44aa538a4d26ae4/A in TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:39,078 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/67ed5cee0f534a88bbdd44d7136c0ddd, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/30a8ad65d21a412385164d71c7fa7949, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/386a11924be94287ae0be47c768a7789] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp, totalSize=35.4 K 2024-11-12T06:47:39,079 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 9af5abf59b704ae5b0f080b92f5b2bf5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731394056770 2024-11-12T06:47:39,079 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67ed5cee0f534a88bbdd44d7136c0ddd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731394056770 2024-11-12T06:47:39,083 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30a8ad65d21a412385164d71c7fa7949, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731394057500 2024-11-12T06:47:39,083 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 00a381ca78ba42248d3b21cffd947c3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731394057500 2024-11-12T06:47:39,084 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 144146f8fd2546a2859fe35bc0608c8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731394058186 2024-11-12T06:47:39,085 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 386a11924be94287ae0be47c768a7789, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731394058186 2024-11-12T06:47:39,113 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0610004ec3b741dc44aa538a4d26ae4#A#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:47:39,113 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0610004ec3b741dc44aa538a4d26ae4#B#compaction#28 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:47:39,114 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/17ad220ddddf4ab08ba80b582323fb90 is 50, key is test_row_0/B:col10/1731394058518/Put/seqid=0 2024-11-12T06:47:39,115 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/dbf076f09c3c413e8a77d9610734393d is 50, key is test_row_0/A:col10/1731394058518/Put/seqid=0 2024-11-12T06:47:39,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741864_1040 (size=12309) 2024-11-12T06:47:39,138 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/17ad220ddddf4ab08ba80b582323fb90 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/17ad220ddddf4ab08ba80b582323fb90 2024-11-12T06:47:39,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741865_1041 (size=12309) 2024-11-12T06:47:39,154 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0610004ec3b741dc44aa538a4d26ae4/B of c0610004ec3b741dc44aa538a4d26ae4 into 17ad220ddddf4ab08ba80b582323fb90(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:47:39,155 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:39,155 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., storeName=c0610004ec3b741dc44aa538a4d26ae4/B, priority=13, startTime=1731394059075; duration=0sec 2024-11-12T06:47:39,155 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:47:39,155 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:B 2024-11-12T06:47:39,155 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:47:39,160 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:47:39,160 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): c0610004ec3b741dc44aa538a4d26ae4/C is initiating minor compaction (all files) 2024-11-12T06:47:39,160 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0610004ec3b741dc44aa538a4d26ae4/C in TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:39,161 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/42a06668792c495294f92006a2d5d8c3, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/8972c365046b447cbbcf9cca9830f047, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/1037b62e62a24375ae0682e2dbdc33fd] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp, totalSize=35.4 K 2024-11-12T06:47:39,162 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 42a06668792c495294f92006a2d5d8c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731394056770 2024-11-12T06:47:39,163 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 8972c365046b447cbbcf9cca9830f047, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731394057500 2024-11-12T06:47:39,165 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 1037b62e62a24375ae0682e2dbdc33fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731394058186 2024-11-12T06:47:39,166 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/dbf076f09c3c413e8a77d9610734393d as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/dbf076f09c3c413e8a77d9610734393d 2024-11-12T06:47:39,178 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0610004ec3b741dc44aa538a4d26ae4/A of c0610004ec3b741dc44aa538a4d26ae4 into dbf076f09c3c413e8a77d9610734393d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:47:39,178 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:39,178 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., storeName=c0610004ec3b741dc44aa538a4d26ae4/A, priority=13, startTime=1731394059075; duration=0sec 2024-11-12T06:47:39,178 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:39,178 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:A 2024-11-12T06:47:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:39,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0610004ec3b741dc44aa538a4d26ae4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T06:47:39,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=A 2024-11-12T06:47:39,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:39,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=B 2024-11-12T06:47:39,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:39,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=C 2024-11-12T06:47:39,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:39,198 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0610004ec3b741dc44aa538a4d26ae4#C#compaction#29 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:47:39,199 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/95597434f0f74640b70a1c3df5aa7ac1 is 50, key is test_row_0/C:col10/1731394058518/Put/seqid=0 2024-11-12T06:47:39,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/706aa1b9770f43388edfeb0c799e138f is 50, key is test_row_0/A:col10/1731394059179/Put/seqid=0 2024-11-12T06:47:39,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T06:47:39,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:39,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:39,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:39,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:39,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T06:47:39,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394119207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:39,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:39,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394119208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:39,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:39,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394119209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:39,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:39,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394119211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:39,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:39,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394119216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:39,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:39,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741866_1042 (size=12309) 2024-11-12T06:47:39,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741867_1043 (size=14541) 2024-11-12T06:47:39,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/706aa1b9770f43388edfeb0c799e138f 2024-11-12T06:47:39,250 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/95597434f0f74640b70a1c3df5aa7ac1 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/95597434f0f74640b70a1c3df5aa7ac1 2024-11-12T06:47:39,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/8e7e4041ca364591b74bccd851b36e65 is 50, key is test_row_0/B:col10/1731394059179/Put/seqid=0 2024-11-12T06:47:39,268 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0610004ec3b741dc44aa538a4d26ae4/C of c0610004ec3b741dc44aa538a4d26ae4 into 95597434f0f74640b70a1c3df5aa7ac1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:47:39,268 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:39,268 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., storeName=c0610004ec3b741dc44aa538a4d26ae4/C, priority=13, startTime=1731394059076; duration=0sec 2024-11-12T06:47:39,268 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:39,268 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:C 2024-11-12T06:47:39,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741868_1044 (size=12151) 2024-11-12T06:47:39,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394119328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:39,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:39,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394119328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:39,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:39,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394119328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:39,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:39,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394119328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:39,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:39,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394119338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:39,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T06:47:39,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:39,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:39,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:39,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:39,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:39,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T06:47:39,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:39,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:39,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:39,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:39,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:39,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394119538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:39,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:39,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:39,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394119538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:39,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:39,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:39,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394119547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:39,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394119547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:39,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:39,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394119547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:39,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:39,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T06:47:39,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:39,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:39,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:39,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:39,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:39,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:39,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/8e7e4041ca364591b74bccd851b36e65 2024-11-12T06:47:39,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/a30409c93d304621a27b720faa112a5c is 50, key is test_row_0/C:col10/1731394059179/Put/seqid=0 2024-11-12T06:47:39,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T06:47:39,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741869_1045 (size=12151) 2024-11-12T06:47:39,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T06:47:39,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:39,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:39,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:39,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:39,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:39,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:39,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394119847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394119847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:39,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:39,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:39,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394119858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:39,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394119858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:39,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:39,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394119858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:39,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:39,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:39,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:39,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:39,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T06:47:39,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:39,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:39,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:39,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:39,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:40,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/a30409c93d304621a27b720faa112a5c 2024-11-12T06:47:40,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T06:47:40,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:40,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/706aa1b9770f43388edfeb0c799e138f as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/706aa1b9770f43388edfeb0c799e138f 2024-11-12T06:47:40,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:40,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:40,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:40,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:40,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/706aa1b9770f43388edfeb0c799e138f, entries=200, sequenceid=141, filesize=14.2 K 2024-11-12T06:47:40,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/8e7e4041ca364591b74bccd851b36e65 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/8e7e4041ca364591b74bccd851b36e65 2024-11-12T06:47:40,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/8e7e4041ca364591b74bccd851b36e65, entries=150, sequenceid=141, filesize=11.9 K 2024-11-12T06:47:40,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/a30409c93d304621a27b720faa112a5c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/a30409c93d304621a27b720faa112a5c 2024-11-12T06:47:40,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/a30409c93d304621a27b720faa112a5c, entries=150, sequenceid=141, filesize=11.9 K 2024-11-12T06:47:40,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c0610004ec3b741dc44aa538a4d26ae4 in 993ms, sequenceid=141, compaction requested=false 2024-11-12T06:47:40,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:40,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T06:47:40,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:40,287 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing c0610004ec3b741dc44aa538a4d26ae4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T06:47:40,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=A 2024-11-12T06:47:40,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:40,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=B 2024-11-12T06:47:40,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:40,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=C 2024-11-12T06:47:40,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:40,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/87bfcca0b3524167bcde3742563f15c2 is 50, key is test_row_0/A:col10/1731394059194/Put/seqid=0 2024-11-12T06:47:40,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741870_1046 (size=12151) 2024-11-12T06:47:40,311 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/87bfcca0b3524167bcde3742563f15c2 2024-11-12T06:47:40,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/d41697b7659d4166a1a6f74543592805 is 50, key is test_row_0/B:col10/1731394059194/Put/seqid=0 2024-11-12T06:47:40,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741871_1047 (size=12151) 2024-11-12T06:47:40,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:40,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:40,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394120385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394120388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:40,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:40,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:40,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394120389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:40,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:40,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394120392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:40,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:40,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394120394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:40,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:40,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394120498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394120498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:40,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:40,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394120498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:40,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:40,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394120507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:40,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:40,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:40,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394120507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:40,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394120707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394120707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394120707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:40,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:40,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394120717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:40,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394120717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T06:47:40,738 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/d41697b7659d4166a1a6f74543592805 2024-11-12T06:47:40,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/db53880fe2a2426194fd31f1ae0e9fed is 50, key is test_row_0/C:col10/1731394059194/Put/seqid=0 2024-11-12T06:47:40,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741872_1048 (size=12151) 2024-11-12T06:47:41,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:41,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394121018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:41,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:41,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394121018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:41,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:41,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:41,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:41,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394121018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:41,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:41,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:41,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394121027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:41,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:41,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:41,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394121027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:41,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:41,154 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/db53880fe2a2426194fd31f1ae0e9fed 2024-11-12T06:47:41,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/87bfcca0b3524167bcde3742563f15c2 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/87bfcca0b3524167bcde3742563f15c2 2024-11-12T06:47:41,175 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/87bfcca0b3524167bcde3742563f15c2, entries=150, sequenceid=157, filesize=11.9 K 2024-11-12T06:47:41,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/d41697b7659d4166a1a6f74543592805 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/d41697b7659d4166a1a6f74543592805 2024-11-12T06:47:41,184 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/d41697b7659d4166a1a6f74543592805, entries=150, sequenceid=157, filesize=11.9 K 2024-11-12T06:47:41,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/db53880fe2a2426194fd31f1ae0e9fed as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/db53880fe2a2426194fd31f1ae0e9fed 2024-11-12T06:47:41,195 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/db53880fe2a2426194fd31f1ae0e9fed, entries=150, sequenceid=157, filesize=11.9 K 2024-11-12T06:47:41,196 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c0610004ec3b741dc44aa538a4d26ae4 in 909ms, sequenceid=157, compaction requested=true 2024-11-12T06:47:41,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:41,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:41,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-12T06:47:41,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-12T06:47:41,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-12T06:47:41,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6160 sec 2024-11-12T06:47:41,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.6280 sec 2024-11-12T06:47:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:41,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0610004ec3b741dc44aa538a4d26ae4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T06:47:41,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=A 2024-11-12T06:47:41,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:41,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=B 2024-11-12T06:47:41,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:41,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=C 2024-11-12T06:47:41,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:41,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/f4e9c9e3ae0143b790d293827ede5283 is 50, key is test_row_0/A:col10/1731394060387/Put/seqid=0 2024-11-12T06:47:41,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741873_1049 (size=14541) 2024-11-12T06:47:41,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:41,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394121573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:41,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394121573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:41,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:41,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:41,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394121574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:41,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:41,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:41,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394121579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:41,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:41,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:41,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394121579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:41,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:41,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394121696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:41,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394121696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:41,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394121696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:41,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:41,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:41,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394121698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:41,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:41,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394121699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:41,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394121907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:41,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394121908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:41,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394121908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394121908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:41,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:41,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394121908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:41,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:41,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:41,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:41,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/f4e9c9e3ae0143b790d293827ede5283 2024-11-12T06:47:41,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/825c59b4ad974881b11f37894cbe9084 is 50, key is test_row_0/B:col10/1731394060387/Put/seqid=0 2024-11-12T06:47:41,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741874_1050 (size=12151) 2024-11-12T06:47:42,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394122217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:42,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:42,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394122217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:42,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:42,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394122217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:42,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394122217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394122217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:42,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:42,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/825c59b4ad974881b11f37894cbe9084 2024-11-12T06:47:42,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/e4a872d499714e54b160aed811158bd8 is 50, key is test_row_0/C:col10/1731394060387/Put/seqid=0 2024-11-12T06:47:42,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741875_1051 (size=12151) 2024-11-12T06:47:42,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/e4a872d499714e54b160aed811158bd8 2024-11-12T06:47:42,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/f4e9c9e3ae0143b790d293827ede5283 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/f4e9c9e3ae0143b790d293827ede5283 2024-11-12T06:47:42,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/f4e9c9e3ae0143b790d293827ede5283, entries=200, sequenceid=175, filesize=14.2 K 2024-11-12T06:47:42,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/825c59b4ad974881b11f37894cbe9084 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/825c59b4ad974881b11f37894cbe9084 2024-11-12T06:47:42,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/825c59b4ad974881b11f37894cbe9084, entries=150, sequenceid=175, filesize=11.9 K 2024-11-12T06:47:42,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/e4a872d499714e54b160aed811158bd8 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/e4a872d499714e54b160aed811158bd8 2024-11-12T06:47:42,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/e4a872d499714e54b160aed811158bd8, entries=150, sequenceid=175, filesize=11.9 K 2024-11-12T06:47:42,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c0610004ec3b741dc44aa538a4d26ae4 in 930ms, sequenceid=175, compaction requested=true 2024-11-12T06:47:42,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:42,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:47:42,460 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:47:42,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:42,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:47:42,460 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:47:42,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:42,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:47:42,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:47:42,464 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:47:42,464 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53542 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:47:42,464 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): c0610004ec3b741dc44aa538a4d26ae4/A is initiating minor compaction (all files) 2024-11-12T06:47:42,464 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): c0610004ec3b741dc44aa538a4d26ae4/B is initiating minor compaction (all files) 2024-11-12T06:47:42,464 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0610004ec3b741dc44aa538a4d26ae4/B in TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:42,464 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0610004ec3b741dc44aa538a4d26ae4/A in TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:42,464 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/17ad220ddddf4ab08ba80b582323fb90, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/8e7e4041ca364591b74bccd851b36e65, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/d41697b7659d4166a1a6f74543592805, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/825c59b4ad974881b11f37894cbe9084] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp, totalSize=47.6 K 2024-11-12T06:47:42,464 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/dbf076f09c3c413e8a77d9610734393d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/706aa1b9770f43388edfeb0c799e138f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/87bfcca0b3524167bcde3742563f15c2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/f4e9c9e3ae0143b790d293827ede5283] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp, totalSize=52.3 K 2024-11-12T06:47:42,466 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 17ad220ddddf4ab08ba80b582323fb90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731394058186 2024-11-12T06:47:42,466 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbf076f09c3c413e8a77d9610734393d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731394058186 2024-11-12T06:47:42,466 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e7e4041ca364591b74bccd851b36e65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731394058543 2024-11-12T06:47:42,466 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 706aa1b9770f43388edfeb0c799e138f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731394058540 2024-11-12T06:47:42,467 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87bfcca0b3524167bcde3742563f15c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731394059194 2024-11-12T06:47:42,467 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting d41697b7659d4166a1a6f74543592805, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731394059194 2024-11-12T06:47:42,467 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4e9c9e3ae0143b790d293827ede5283, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731394060387 2024-11-12T06:47:42,468 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 825c59b4ad974881b11f37894cbe9084, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731394060387 2024-11-12T06:47:42,486 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0610004ec3b741dc44aa538a4d26ae4#A#compaction#39 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:47:42,487 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/8f1d1bf68e0d462b8a87df47ae2ffacc is 50, key is test_row_0/A:col10/1731394060387/Put/seqid=0 2024-11-12T06:47:42,493 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0610004ec3b741dc44aa538a4d26ae4#B#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:47:42,494 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/478ebae8b5434a5f8e49fd3ad58b399b is 50, key is test_row_0/B:col10/1731394060387/Put/seqid=0 2024-11-12T06:47:42,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741876_1052 (size=12595) 2024-11-12T06:47:42,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741877_1053 (size=12595) 2024-11-12T06:47:42,514 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/478ebae8b5434a5f8e49fd3ad58b399b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/478ebae8b5434a5f8e49fd3ad58b399b 2024-11-12T06:47:42,525 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0610004ec3b741dc44aa538a4d26ae4/B of c0610004ec3b741dc44aa538a4d26ae4 into 478ebae8b5434a5f8e49fd3ad58b399b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:47:42,526 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:42,526 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., storeName=c0610004ec3b741dc44aa538a4d26ae4/B, priority=12, startTime=1731394062460; duration=0sec 2024-11-12T06:47:42,526 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:47:42,526 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:B 2024-11-12T06:47:42,526 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:47:42,529 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:47:42,529 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): c0610004ec3b741dc44aa538a4d26ae4/C is initiating minor compaction (all files) 2024-11-12T06:47:42,529 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0610004ec3b741dc44aa538a4d26ae4/C in TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:42,529 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/95597434f0f74640b70a1c3df5aa7ac1, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/a30409c93d304621a27b720faa112a5c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/db53880fe2a2426194fd31f1ae0e9fed, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/e4a872d499714e54b160aed811158bd8] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp, totalSize=47.6 K 2024-11-12T06:47:42,530 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 95597434f0f74640b70a1c3df5aa7ac1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731394058186 2024-11-12T06:47:42,530 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting a30409c93d304621a27b720faa112a5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731394058543 2024-11-12T06:47:42,531 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting db53880fe2a2426194fd31f1ae0e9fed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731394059194 2024-11-12T06:47:42,531 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting e4a872d499714e54b160aed811158bd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731394060387 2024-11-12T06:47:42,555 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0610004ec3b741dc44aa538a4d26ae4#C#compaction#41 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:47:42,556 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/ab1eee51794e49a58f555c8794ff83b8 is 50, key is test_row_0/C:col10/1731394060387/Put/seqid=0 2024-11-12T06:47:42,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741878_1054 (size=12595) 2024-11-12T06:47:42,577 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/ab1eee51794e49a58f555c8794ff83b8 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/ab1eee51794e49a58f555c8794ff83b8 2024-11-12T06:47:42,588 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0610004ec3b741dc44aa538a4d26ae4/C of c0610004ec3b741dc44aa538a4d26ae4 into ab1eee51794e49a58f555c8794ff83b8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:47:42,588 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:42,589 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., storeName=c0610004ec3b741dc44aa538a4d26ae4/C, priority=12, startTime=1731394062460; duration=0sec 2024-11-12T06:47:42,589 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:42,589 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:C 2024-11-12T06:47:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:42,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0610004ec3b741dc44aa538a4d26ae4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T06:47:42,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=A 2024-11-12T06:47:42,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:42,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=B 2024-11-12T06:47:42,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:42,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=C 2024-11-12T06:47:42,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T06:47:42,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/c2be7ade369948369ac90b320676452e is 50, key is test_row_0/A:col10/1731394061549/Put/seqid=0 2024-11-12T06:47:42,738 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:47:42,738 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:47:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-12T06:47:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T06:47:42,741 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:47:42,743 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:47:42,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:47:42,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394122754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:42,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:42,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:42,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394122757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:42,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:42,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741879_1055 (size=12151) 2024-11-12T06:47:42,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394122759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:42,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394122759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:42,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394122759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T06:47:42,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:42,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394122867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:42,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:42,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:42,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394122867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:42,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:42,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394122877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:42,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394122877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394122877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:42,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T06:47:42,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:42,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:42,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:42,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:42,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:42,905 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/8f1d1bf68e0d462b8a87df47ae2ffacc as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/8f1d1bf68e0d462b8a87df47ae2ffacc 2024-11-12T06:47:42,913 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0610004ec3b741dc44aa538a4d26ae4/A of c0610004ec3b741dc44aa538a4d26ae4 into 8f1d1bf68e0d462b8a87df47ae2ffacc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:47:42,913 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:42,913 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., storeName=c0610004ec3b741dc44aa538a4d26ae4/A, priority=12, startTime=1731394062460; duration=0sec 2024-11-12T06:47:42,913 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:42,913 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:A 2024-11-12T06:47:43,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T06:47:43,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:43,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:43,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:43,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:43,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T06:47:43,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:43,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394123077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:43,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:43,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:43,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394123078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:43,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394123087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:43,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:43,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394123087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:43,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394123088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:43,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/c2be7ade369948369ac90b320676452e 2024-11-12T06:47:43,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/442f39eb7c84495eb7b8e33f1ac0ff1e is 50, key is test_row_0/B:col10/1731394061549/Put/seqid=0 2024-11-12T06:47:43,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741880_1056 (size=12151) 2024-11-12T06:47:43,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T06:47:43,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:43,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:43,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:43,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:43,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:43,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T06:47:43,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:43,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:43,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:43,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:43,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:43,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T06:47:43,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:43,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394123387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:43,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:43,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:43,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394123387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:43,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:43,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:43,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394123397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:43,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:43,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394123397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:43,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394123397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:43,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:43,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:43,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T06:47:43,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:43,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:43,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:43,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:43,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:43,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:43,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/442f39eb7c84495eb7b8e33f1ac0ff1e 2024-11-12T06:47:43,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/c04778e836b24612a1a983a88c9b9668 is 50, key is test_row_0/C:col10/1731394061549/Put/seqid=0 2024-11-12T06:47:43,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741881_1057 (size=12151) 2024-11-12T06:47:43,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T06:47:43,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:43,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:43,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:43,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:43,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:43,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T06:47:43,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:43,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:43,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:43,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:43,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T06:47:43,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394123897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:43,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:43,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394123897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:43,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:43,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394123907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:43,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:43,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394123907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394123907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:43,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T06:47:43,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:43,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:43,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:43,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:43,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:44,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/c04778e836b24612a1a983a88c9b9668 2024-11-12T06:47:44,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/c2be7ade369948369ac90b320676452e as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/c2be7ade369948369ac90b320676452e 2024-11-12T06:47:44,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/c2be7ade369948369ac90b320676452e, entries=150, sequenceid=193, filesize=11.9 K 2024-11-12T06:47:44,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/442f39eb7c84495eb7b8e33f1ac0ff1e as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/442f39eb7c84495eb7b8e33f1ac0ff1e 2024-11-12T06:47:44,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/442f39eb7c84495eb7b8e33f1ac0ff1e, entries=150, sequenceid=193, filesize=11.9 K 2024-11-12T06:47:44,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/c04778e836b24612a1a983a88c9b9668 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/c04778e836b24612a1a983a88c9b9668 2024-11-12T06:47:44,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/c04778e836b24612a1a983a88c9b9668, entries=150, sequenceid=193, filesize=11.9 K 2024-11-12T06:47:44,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=100.63 KB/103050 for c0610004ec3b741dc44aa538a4d26ae4 in 1325ms, sequenceid=193, compaction requested=false 2024-11-12T06:47:44,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:44,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T06:47:44,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:44,130 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing c0610004ec3b741dc44aa538a4d26ae4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T06:47:44,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=A 2024-11-12T06:47:44,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:44,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=B 2024-11-12T06:47:44,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:44,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=C 2024-11-12T06:47:44,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:44,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/e7c37cb8fcd944428976c225898d4f06 is 50, key is test_row_0/A:col10/1731394062754/Put/seqid=0 2024-11-12T06:47:44,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741882_1058 (size=12151) 2024-11-12T06:47:44,142 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/e7c37cb8fcd944428976c225898d4f06 2024-11-12T06:47:44,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/07ecae6e10b34f6c9e8ab63b3afc0f2d is 50, key is test_row_0/B:col10/1731394062754/Put/seqid=0 2024-11-12T06:47:44,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741883_1059 (size=12151) 2024-11-12T06:47:44,562 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/07ecae6e10b34f6c9e8ab63b3afc0f2d 2024-11-12T06:47:44,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/a6a0a2c75760452eadeefcb89ed18aac is 50, key is test_row_0/C:col10/1731394062754/Put/seqid=0 2024-11-12T06:47:44,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741884_1060 (size=12151) 2024-11-12T06:47:44,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T06:47:44,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:44,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:44,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:44,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394124927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:44,936 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_0' in c0610004ec3b741dc44aa538a4d26ae4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:47:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:44,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:44,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:44,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394124929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:44,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394124930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:44,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:44,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394124933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:44,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:44,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394124933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:44,976 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/a6a0a2c75760452eadeefcb89ed18aac 2024-11-12T06:47:44,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/e7c37cb8fcd944428976c225898d4f06 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/e7c37cb8fcd944428976c225898d4f06 2024-11-12T06:47:44,988 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/e7c37cb8fcd944428976c225898d4f06, entries=150, sequenceid=212, filesize=11.9 K 2024-11-12T06:47:44,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/07ecae6e10b34f6c9e8ab63b3afc0f2d as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/07ecae6e10b34f6c9e8ab63b3afc0f2d 2024-11-12T06:47:44,995 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/07ecae6e10b34f6c9e8ab63b3afc0f2d, entries=150, sequenceid=212, filesize=11.9 K 2024-11-12T06:47:44,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/a6a0a2c75760452eadeefcb89ed18aac as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/a6a0a2c75760452eadeefcb89ed18aac 2024-11-12T06:47:45,002 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/a6a0a2c75760452eadeefcb89ed18aac, entries=150, sequenceid=212, filesize=11.9 K 2024-11-12T06:47:45,003 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for c0610004ec3b741dc44aa538a4d26ae4 in 873ms, sequenceid=212, compaction requested=true 2024-11-12T06:47:45,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:45,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:45,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-12T06:47:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-12T06:47:45,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-12T06:47:45,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2620 sec 2024-11-12T06:47:45,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.2700 sec 2024-11-12T06:47:45,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:45,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0610004ec3b741dc44aa538a4d26ae4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T06:47:45,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=A 2024-11-12T06:47:45,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:45,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=B 2024-11-12T06:47:45,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:45,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=C 2024-11-12T06:47:45,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:45,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/e0bbbf8995fd408fbb02ec20ec1e6ec5 is 50, key is test_row_0/A:col10/1731394065051/Put/seqid=0 2024-11-12T06:47:45,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741885_1061 (size=12151) 2024-11-12T06:47:45,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:45,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394125074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:45,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:45,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394125075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:45,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:45,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394125075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:45,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:45,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394125077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:45,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:45,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394125187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394125188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:45,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:45,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394125188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:45,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394125188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:45,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394125397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:45,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:45,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394125397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:45,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:45,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:45,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394125397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:45,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:45,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:45,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394125397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:45,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:45,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:45,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/e0bbbf8995fd408fbb02ec20ec1e6ec5 2024-11-12T06:47:45,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/0da492c92db34c799346be197b4dc8b8 is 50, key is test_row_0/B:col10/1731394065051/Put/seqid=0 2024-11-12T06:47:45,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741886_1062 (size=12151) 2024-11-12T06:47:45,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/0da492c92db34c799346be197b4dc8b8 2024-11-12T06:47:45,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/7a6bd2b0d63f4040a7b9d73bc1ddfc81 is 50, key is test_row_0/C:col10/1731394065051/Put/seqid=0 2024-11-12T06:47:45,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741887_1063 (size=12151) 2024-11-12T06:47:45,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394125707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:45,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394125707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:45,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:45,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394125708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:45,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:45,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394125708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:45,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:45,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:45,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/7a6bd2b0d63f4040a7b9d73bc1ddfc81 2024-11-12T06:47:45,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/e0bbbf8995fd408fbb02ec20ec1e6ec5 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/e0bbbf8995fd408fbb02ec20ec1e6ec5 2024-11-12T06:47:45,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/e0bbbf8995fd408fbb02ec20ec1e6ec5, entries=150, sequenceid=228, filesize=11.9 K 2024-11-12T06:47:45,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/0da492c92db34c799346be197b4dc8b8 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/0da492c92db34c799346be197b4dc8b8 2024-11-12T06:47:45,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/0da492c92db34c799346be197b4dc8b8, entries=150, sequenceid=228, filesize=11.9 K 2024-11-12T06:47:45,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/7a6bd2b0d63f4040a7b9d73bc1ddfc81 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/7a6bd2b0d63f4040a7b9d73bc1ddfc81 2024-11-12T06:47:45,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/7a6bd2b0d63f4040a7b9d73bc1ddfc81, entries=150, sequenceid=228, filesize=11.9 K 2024-11-12T06:47:45,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c0610004ec3b741dc44aa538a4d26ae4 in 881ms, sequenceid=228, compaction requested=true 2024-11-12T06:47:45,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:45,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:47:45,933 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:47:45,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:45,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:47:45,933 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:47:45,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:45,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:47:45,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:47:45,935 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:47:45,935 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:47:45,935 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): c0610004ec3b741dc44aa538a4d26ae4/B is initiating minor compaction (all files) 2024-11-12T06:47:45,935 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): c0610004ec3b741dc44aa538a4d26ae4/A is initiating minor compaction (all files) 2024-11-12T06:47:45,935 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0610004ec3b741dc44aa538a4d26ae4/A in TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:45,935 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0610004ec3b741dc44aa538a4d26ae4/B in TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:45,935 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/478ebae8b5434a5f8e49fd3ad58b399b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/442f39eb7c84495eb7b8e33f1ac0ff1e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/07ecae6e10b34f6c9e8ab63b3afc0f2d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/0da492c92db34c799346be197b4dc8b8] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp, totalSize=47.9 K 2024-11-12T06:47:45,935 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/8f1d1bf68e0d462b8a87df47ae2ffacc, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/c2be7ade369948369ac90b320676452e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/e7c37cb8fcd944428976c225898d4f06, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/e0bbbf8995fd408fbb02ec20ec1e6ec5] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp, totalSize=47.9 K 2024-11-12T06:47:45,936 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 478ebae8b5434a5f8e49fd3ad58b399b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731394060387 2024-11-12T06:47:45,936 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f1d1bf68e0d462b8a87df47ae2ffacc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731394060387 2024-11-12T06:47:45,936 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 442f39eb7c84495eb7b8e33f1ac0ff1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731394061549 2024-11-12T06:47:45,936 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2be7ade369948369ac90b320676452e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731394061549 2024-11-12T06:47:45,936 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 07ecae6e10b34f6c9e8ab63b3afc0f2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731394062754 2024-11-12T06:47:45,936 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7c37cb8fcd944428976c225898d4f06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731394062754 2024-11-12T06:47:45,937 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 0da492c92db34c799346be197b4dc8b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731394064930 2024-11-12T06:47:45,937 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0bbbf8995fd408fbb02ec20ec1e6ec5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731394064930 2024-11-12T06:47:45,947 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0610004ec3b741dc44aa538a4d26ae4#A#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:47:45,947 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0610004ec3b741dc44aa538a4d26ae4#B#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:47:45,948 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/a6d90d23ad85496c90190cd3c6b4915b is 50, key is test_row_0/A:col10/1731394065051/Put/seqid=0 2024-11-12T06:47:45,948 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/7d4c3bf5a71b4e27a57df1829f26a348 is 50, key is test_row_0/B:col10/1731394065051/Put/seqid=0 2024-11-12T06:47:45,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741888_1064 (size=12731) 2024-11-12T06:47:45,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741889_1065 (size=12731) 2024-11-12T06:47:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:46,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0610004ec3b741dc44aa538a4d26ae4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T06:47:46,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=A 2024-11-12T06:47:46,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:46,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=B 2024-11-12T06:47:46,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:46,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=C 2024-11-12T06:47:46,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:46,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/33412a58247c4612a6f32195dde3cf5b is 50, key is test_row_0/A:col10/1731394065073/Put/seqid=0 2024-11-12T06:47:46,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741890_1066 (size=14541) 2024-11-12T06:47:46,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/33412a58247c4612a6f32195dde3cf5b 2024-11-12T06:47:46,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/805fc7f0b9c24cb091e8431c9a3c2ced is 50, key is test_row_0/B:col10/1731394065073/Put/seqid=0 2024-11-12T06:47:46,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:46,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394126244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:46,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394126245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:46,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394126247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:46,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:46,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394126251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:46,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:46,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741891_1067 (size=12151) 2024-11-12T06:47:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:46,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394126357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:46,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394126357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:46,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:46,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394126358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:46,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:46,365 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/a6d90d23ad85496c90190cd3c6b4915b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/a6d90d23ad85496c90190cd3c6b4915b 2024-11-12T06:47:46,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394126367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:46,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:46,375 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/7d4c3bf5a71b4e27a57df1829f26a348 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/7d4c3bf5a71b4e27a57df1829f26a348 2024-11-12T06:47:46,379 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0610004ec3b741dc44aa538a4d26ae4/A of c0610004ec3b741dc44aa538a4d26ae4 into a6d90d23ad85496c90190cd3c6b4915b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:47:46,379 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:46,379 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., storeName=c0610004ec3b741dc44aa538a4d26ae4/A, priority=12, startTime=1731394065933; duration=0sec 2024-11-12T06:47:46,379 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:47:46,379 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:A 2024-11-12T06:47:46,380 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:47:46,382 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:47:46,382 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): c0610004ec3b741dc44aa538a4d26ae4/C is initiating minor compaction (all files) 2024-11-12T06:47:46,383 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0610004ec3b741dc44aa538a4d26ae4/C in TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:46,383 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/ab1eee51794e49a58f555c8794ff83b8, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/c04778e836b24612a1a983a88c9b9668, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/a6a0a2c75760452eadeefcb89ed18aac, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/7a6bd2b0d63f4040a7b9d73bc1ddfc81] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp, totalSize=47.9 K 2024-11-12T06:47:46,383 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0610004ec3b741dc44aa538a4d26ae4/B of c0610004ec3b741dc44aa538a4d26ae4 into 7d4c3bf5a71b4e27a57df1829f26a348(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:47:46,383 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:46,383 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., storeName=c0610004ec3b741dc44aa538a4d26ae4/B, priority=12, startTime=1731394065933; duration=0sec 2024-11-12T06:47:46,383 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:46,383 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:B 2024-11-12T06:47:46,383 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab1eee51794e49a58f555c8794ff83b8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731394060387 2024-11-12T06:47:46,384 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting c04778e836b24612a1a983a88c9b9668, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731394061549 2024-11-12T06:47:46,385 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6a0a2c75760452eadeefcb89ed18aac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731394062754 2024-11-12T06:47:46,385 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a6bd2b0d63f4040a7b9d73bc1ddfc81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731394064930 2024-11-12T06:47:46,400 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0610004ec3b741dc44aa538a4d26ae4#C#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:47:46,401 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/30aa4aceaf954859b3209a9424dc23d3 is 50, key is test_row_0/C:col10/1731394065051/Put/seqid=0 2024-11-12T06:47:46,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741892_1068 (size=12731) 2024-11-12T06:47:46,418 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/30aa4aceaf954859b3209a9424dc23d3 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/30aa4aceaf954859b3209a9424dc23d3 2024-11-12T06:47:46,428 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0610004ec3b741dc44aa538a4d26ae4/C of c0610004ec3b741dc44aa538a4d26ae4 into 30aa4aceaf954859b3209a9424dc23d3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:47:46,428 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:46,428 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., storeName=c0610004ec3b741dc44aa538a4d26ae4/C, priority=12, startTime=1731394065934; duration=0sec 2024-11-12T06:47:46,428 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:46,429 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:C 2024-11-12T06:47:46,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:46,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394126567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:46,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:46,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394126567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:46,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:46,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394126567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:46,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:46,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:46,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394126578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:46,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:46,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/805fc7f0b9c24cb091e8431c9a3c2ced 2024-11-12T06:47:46,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/9f3a449b41d64f5d8d713fc398f845ff is 50, key is test_row_0/C:col10/1731394065073/Put/seqid=0 2024-11-12T06:47:46,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741893_1069 (size=12151) 2024-11-12T06:47:46,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:46,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394126877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:46,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394126877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:46,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:46,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394126877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:46,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394126887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:46,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T06:47:46,908 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:47:46,908 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:47:46,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-12T06:47:46,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T06:47:46,910 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:47:46,911 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:47:46,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:47:46,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:46,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394126957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:46,960 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_0' in c0610004ec3b741dc44aa538a4d26ae4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:47:46,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:46,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:47,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T06:47:47,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T06:47:47,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:47,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:47,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:47,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:47,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:47,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:47,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/9f3a449b41d64f5d8d713fc398f845ff 2024-11-12T06:47:47,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/33412a58247c4612a6f32195dde3cf5b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/33412a58247c4612a6f32195dde3cf5b 2024-11-12T06:47:47,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/33412a58247c4612a6f32195dde3cf5b, entries=200, sequenceid=245, filesize=14.2 K 2024-11-12T06:47:47,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/805fc7f0b9c24cb091e8431c9a3c2ced as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/805fc7f0b9c24cb091e8431c9a3c2ced 2024-11-12T06:47:47,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/805fc7f0b9c24cb091e8431c9a3c2ced, entries=150, sequenceid=245, filesize=11.9 K 2024-11-12T06:47:47,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/9f3a449b41d64f5d8d713fc398f845ff as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/9f3a449b41d64f5d8d713fc398f845ff 2024-11-12T06:47:47,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/9f3a449b41d64f5d8d713fc398f845ff, entries=150, sequenceid=245, filesize=11.9 K 2024-11-12T06:47:47,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c0610004ec3b741dc44aa538a4d26ae4 in 934ms, sequenceid=245, compaction requested=false 2024-11-12T06:47:47,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:47,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T06:47:47,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:47,218 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing c0610004ec3b741dc44aa538a4d26ae4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T06:47:47,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=A 2024-11-12T06:47:47,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:47,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=B 2024-11-12T06:47:47,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:47,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=C 2024-11-12T06:47:47,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:47,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/cf8a5358f5864b9d9335cd7a54e34faa is 50, key is test_row_0/A:col10/1731394066249/Put/seqid=0 2024-11-12T06:47:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T06:47:47,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741894_1070 (size=12251) 2024-11-12T06:47:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:47,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:47,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394127413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:47,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394127414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394127414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:47,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394127415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:47,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394127527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:47,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:47,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394127527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:47,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:47,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394127528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:47,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:47,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394127528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T06:47:47,630 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/cf8a5358f5864b9d9335cd7a54e34faa 2024-11-12T06:47:47,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/f88c014154cd449d92566d5585f07e2f is 50, key is test_row_0/B:col10/1731394066249/Put/seqid=0 2024-11-12T06:47:47,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741895_1071 (size=12251) 2024-11-12T06:47:47,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:47,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394127738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394127738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394127738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:47,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394127747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:48,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T06:47:48,048 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/f88c014154cd449d92566d5585f07e2f 2024-11-12T06:47:48,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:48,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394128047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:48,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394128047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394128047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:48,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/11714193cd3a428484336554bd009e0b is 50, key is test_row_0/C:col10/1731394066249/Put/seqid=0 2024-11-12T06:47:48,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741896_1072 (size=12251) 2024-11-12T06:47:48,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:48,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394128058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:48,463 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/11714193cd3a428484336554bd009e0b 2024-11-12T06:47:48,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/cf8a5358f5864b9d9335cd7a54e34faa as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/cf8a5358f5864b9d9335cd7a54e34faa 2024-11-12T06:47:48,475 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/cf8a5358f5864b9d9335cd7a54e34faa, entries=150, sequenceid=263, filesize=12.0 K 2024-11-12T06:47:48,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/f88c014154cd449d92566d5585f07e2f as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/f88c014154cd449d92566d5585f07e2f 2024-11-12T06:47:48,485 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/f88c014154cd449d92566d5585f07e2f, entries=150, sequenceid=263, filesize=12.0 K 2024-11-12T06:47:48,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/11714193cd3a428484336554bd009e0b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/11714193cd3a428484336554bd009e0b 2024-11-12T06:47:48,492 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/11714193cd3a428484336554bd009e0b, entries=150, sequenceid=263, filesize=12.0 K 2024-11-12T06:47:48,493 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for c0610004ec3b741dc44aa538a4d26ae4 in 1275ms, sequenceid=263, compaction requested=true 2024-11-12T06:47:48,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:48,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:48,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-12T06:47:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-12T06:47:48,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-12T06:47:48,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5840 sec 2024-11-12T06:47:48,501 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.5920 sec 2024-11-12T06:47:48,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:48,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0610004ec3b741dc44aa538a4d26ae4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T06:47:48,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=A 2024-11-12T06:47:48,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:47:48,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=B 2024-11-12T06:47:48,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:47:48,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=C 2024-11-12T06:47:48,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:47:48,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/b0dedabb9478428195d054a595e4532b is 50, key is test_row_0/A:col10/1731394068563/Put/seqid=0 2024-11-12T06:47:48,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741897_1073 (size=12301) 2024-11-12T06:47:48,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394128607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:48,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:48,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394128607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:48,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:48,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:48,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394128614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:48,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:48,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394128616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:48,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:48,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:48,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394128727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:48,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:48,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394128727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:48,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:48,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:48,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:48,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394128737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:48,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394128737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:48,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:48,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:48,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394128937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:48,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:48,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:48,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394128937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:48,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:48,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394128948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:48,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:48,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394128948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:49,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/b0dedabb9478428195d054a595e4532b 2024-11-12T06:47:49,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/b56628d53e9e41129ea27d34d63f5ef1 is 50, key is test_row_0/B:col10/1731394068563/Put/seqid=0 2024-11-12T06:47:49,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T06:47:49,058 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:47:49,058 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:47:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-12T06:47:49,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T06:47:49,061 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:47:49,062 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:47:49,063 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:47:49,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741898_1074 (size=12301) 2024-11-12T06:47:49,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/b56628d53e9e41129ea27d34d63f5ef1 2024-11-12T06:47:49,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/6325cf4c39884dbeba4d8be320d0c614 is 50, key is test_row_0/C:col10/1731394068563/Put/seqid=0 2024-11-12T06:47:49,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741899_1075 (size=12301) 2024-11-12T06:47:49,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T06:47:49,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T06:47:49,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:49,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:49,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:49,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:49,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:49,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:49,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:49,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394129247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:49,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:49,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:49,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394129247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:49,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:49,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:49,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394129267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:49,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394129267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:49,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T06:47:49,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:49,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:49,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:49,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:49,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:49,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:49,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T06:47:49,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/6325cf4c39884dbeba4d8be320d0c614 2024-11-12T06:47:49,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/b0dedabb9478428195d054a595e4532b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/b0dedabb9478428195d054a595e4532b 2024-11-12T06:47:49,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/b0dedabb9478428195d054a595e4532b, entries=150, sequenceid=284, filesize=12.0 K 2024-11-12T06:47:49,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/b56628d53e9e41129ea27d34d63f5ef1 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/b56628d53e9e41129ea27d34d63f5ef1 2024-11-12T06:47:49,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T06:47:49,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:49,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:49,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:49,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:49,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:49,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/b56628d53e9e41129ea27d34d63f5ef1, entries=150, sequenceid=284, filesize=12.0 K 2024-11-12T06:47:49,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/6325cf4c39884dbeba4d8be320d0c614 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/6325cf4c39884dbeba4d8be320d0c614 2024-11-12T06:47:49,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/6325cf4c39884dbeba4d8be320d0c614, entries=150, sequenceid=284, filesize=12.0 K 2024-11-12T06:47:49,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for c0610004ec3b741dc44aa538a4d26ae4 in 984ms, sequenceid=284, compaction requested=true 2024-11-12T06:47:49,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:49,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:47:49,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:49,548 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:47:49,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:47:49,548 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:47:49,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:49,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:47:49,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:47:49,550 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51824 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:47:49,550 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49434 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:47:49,550 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): c0610004ec3b741dc44aa538a4d26ae4/A is initiating minor compaction (all files) 2024-11-12T06:47:49,550 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): c0610004ec3b741dc44aa538a4d26ae4/B is initiating minor compaction (all files) 2024-11-12T06:47:49,550 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0610004ec3b741dc44aa538a4d26ae4/A in TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:49,550 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0610004ec3b741dc44aa538a4d26ae4/B in TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:49,550 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/7d4c3bf5a71b4e27a57df1829f26a348, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/805fc7f0b9c24cb091e8431c9a3c2ced, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/f88c014154cd449d92566d5585f07e2f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/b56628d53e9e41129ea27d34d63f5ef1] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp, totalSize=48.3 K 2024-11-12T06:47:49,550 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/a6d90d23ad85496c90190cd3c6b4915b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/33412a58247c4612a6f32195dde3cf5b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/cf8a5358f5864b9d9335cd7a54e34faa, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/b0dedabb9478428195d054a595e4532b] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp, totalSize=50.6 K 2024-11-12T06:47:49,551 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d4c3bf5a71b4e27a57df1829f26a348, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731394064930 2024-11-12T06:47:49,552 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6d90d23ad85496c90190cd3c6b4915b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731394064930 2024-11-12T06:47:49,552 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 805fc7f0b9c24cb091e8431c9a3c2ced, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731394065071 2024-11-12T06:47:49,552 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33412a58247c4612a6f32195dde3cf5b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731394065071 2024-11-12T06:47:49,552 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting f88c014154cd449d92566d5585f07e2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731394066231 2024-11-12T06:47:49,553 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf8a5358f5864b9d9335cd7a54e34faa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731394066231 2024-11-12T06:47:49,553 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting b56628d53e9e41129ea27d34d63f5ef1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731394068562 2024-11-12T06:47:49,553 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0dedabb9478428195d054a595e4532b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731394068562 2024-11-12T06:47:49,572 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0610004ec3b741dc44aa538a4d26ae4#A#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:47:49,572 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/f77f7747c75841d3ab7c4b60b8beed4a is 50, key is test_row_0/A:col10/1731394068563/Put/seqid=0 2024-11-12T06:47:49,575 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0610004ec3b741dc44aa538a4d26ae4#B#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:47:49,575 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/c578aab538a04db59879961f84fc8e8d is 50, key is test_row_0/B:col10/1731394068563/Put/seqid=0 2024-11-12T06:47:49,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741900_1076 (size=13017) 2024-11-12T06:47:49,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741901_1077 (size=13017) 2024-11-12T06:47:49,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T06:47:49,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:49,682 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing c0610004ec3b741dc44aa538a4d26ae4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T06:47:49,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=A 2024-11-12T06:47:49,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:49,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=B 2024-11-12T06:47:49,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:49,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=C 2024-11-12T06:47:49,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T06:47:49,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/5a1894ecb3234b9d9046201b66da2850 is 50, key is test_row_0/A:col10/1731394068612/Put/seqid=0 2024-11-12T06:47:49,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741902_1078 (size=12301) 2024-11-12T06:47:49,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:49,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394129796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:49,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:49,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394129797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:49,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:49,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394129803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:49,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394129805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:49,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394129918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:49,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:49,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394129918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:49,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:49,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:49,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394129919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394129919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:49,994 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/c578aab538a04db59879961f84fc8e8d as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/c578aab538a04db59879961f84fc8e8d 2024-11-12T06:47:49,995 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/f77f7747c75841d3ab7c4b60b8beed4a as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/f77f7747c75841d3ab7c4b60b8beed4a 2024-11-12T06:47:50,001 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0610004ec3b741dc44aa538a4d26ae4/A of c0610004ec3b741dc44aa538a4d26ae4 into f77f7747c75841d3ab7c4b60b8beed4a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:47:50,001 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:50,001 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., storeName=c0610004ec3b741dc44aa538a4d26ae4/A, priority=12, startTime=1731394069548; duration=0sec 2024-11-12T06:47:50,001 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:47:50,002 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:A 2024-11-12T06:47:50,002 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:47:50,003 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49434 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:47:50,003 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): c0610004ec3b741dc44aa538a4d26ae4/C is initiating minor compaction (all files) 2024-11-12T06:47:50,003 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0610004ec3b741dc44aa538a4d26ae4/C in TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:50,003 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/30aa4aceaf954859b3209a9424dc23d3, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/9f3a449b41d64f5d8d713fc398f845ff, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/11714193cd3a428484336554bd009e0b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/6325cf4c39884dbeba4d8be320d0c614] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp, totalSize=48.3 K 2024-11-12T06:47:50,004 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30aa4aceaf954859b3209a9424dc23d3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731394064930 2024-11-12T06:47:50,004 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f3a449b41d64f5d8d713fc398f845ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731394065071 2024-11-12T06:47:50,004 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11714193cd3a428484336554bd009e0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731394066231 2024-11-12T06:47:50,005 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6325cf4c39884dbeba4d8be320d0c614, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731394068562 2024-11-12T06:47:50,007 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0610004ec3b741dc44aa538a4d26ae4/B of c0610004ec3b741dc44aa538a4d26ae4 into c578aab538a04db59879961f84fc8e8d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:47:50,007 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:50,007 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., storeName=c0610004ec3b741dc44aa538a4d26ae4/B, priority=12, startTime=1731394069548; duration=0sec 2024-11-12T06:47:50,007 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:50,007 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:B 2024-11-12T06:47:50,022 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0610004ec3b741dc44aa538a4d26ae4#C#compaction#66 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:47:50,022 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/155c37788e224295ac60bdf46bfb6238 is 50, key is test_row_0/C:col10/1731394068563/Put/seqid=0 2024-11-12T06:47:50,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741903_1079 (size=13017) 2024-11-12T06:47:50,042 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/155c37788e224295ac60bdf46bfb6238 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/155c37788e224295ac60bdf46bfb6238 2024-11-12T06:47:50,051 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0610004ec3b741dc44aa538a4d26ae4/C of c0610004ec3b741dc44aa538a4d26ae4 into 155c37788e224295ac60bdf46bfb6238(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:47:50,051 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:50,051 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., storeName=c0610004ec3b741dc44aa538a4d26ae4/C, priority=12, startTime=1731394069549; duration=0sec 2024-11-12T06:47:50,051 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:50,051 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:C 2024-11-12T06:47:50,094 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/5a1894ecb3234b9d9046201b66da2850 2024-11-12T06:47:50,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/62bf03d7b0a9402fadfbd1b2a5cc7df5 is 50, key is test_row_0/B:col10/1731394068612/Put/seqid=0 2024-11-12T06:47:50,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:50,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394130127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:50,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:50,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:50,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:50,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741904_1080 (size=12301) 2024-11-12T06:47:50,132 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/62bf03d7b0a9402fadfbd1b2a5cc7df5 2024-11-12T06:47:50,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394130131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:50,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394130138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:50,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394130138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:50,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:50,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:50,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:50,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:50,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:50,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/93ebed1c02184cff8f7e561f6762d966 is 50, key is test_row_0/C:col10/1731394068612/Put/seqid=0 2024-11-12T06:47:50,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741905_1081 (size=12301) 2024-11-12T06:47:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T06:47:50,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:50,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394130437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:50,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:50,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:50,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:50,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:50,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394130447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:50,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:50,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394130447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:50,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:50,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394130447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:50,553 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/93ebed1c02184cff8f7e561f6762d966 2024-11-12T06:47:50,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/5a1894ecb3234b9d9046201b66da2850 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/5a1894ecb3234b9d9046201b66da2850 2024-11-12T06:47:50,567 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/5a1894ecb3234b9d9046201b66da2850, entries=150, sequenceid=296, filesize=12.0 K 2024-11-12T06:47:50,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/62bf03d7b0a9402fadfbd1b2a5cc7df5 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/62bf03d7b0a9402fadfbd1b2a5cc7df5 2024-11-12T06:47:50,573 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/62bf03d7b0a9402fadfbd1b2a5cc7df5, entries=150, sequenceid=296, filesize=12.0 K 2024-11-12T06:47:50,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/93ebed1c02184cff8f7e561f6762d966 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/93ebed1c02184cff8f7e561f6762d966 2024-11-12T06:47:50,580 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/93ebed1c02184cff8f7e561f6762d966, entries=150, sequenceid=296, filesize=12.0 K 2024-11-12T06:47:50,582 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c0610004ec3b741dc44aa538a4d26ae4 in 900ms, sequenceid=296, compaction requested=false 2024-11-12T06:47:50,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:50,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:50,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-12T06:47:50,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-12T06:47:50,591 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-12T06:47:50,591 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5220 sec 2024-11-12T06:47:50,594 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.5340 sec 2024-11-12T06:47:50,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394069795, max timestamp=1731394070947Num uniques 859; 2024-11-12T06:47:50,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394069795, max timestamp=1731394070947Num uniques 859; 2024-11-12T06:47:50,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:47:50,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:47:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:50,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0610004ec3b741dc44aa538a4d26ae4 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-12T06:47:50,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=A 2024-11-12T06:47:50,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:47:50,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394069795, max timestamp=1731394070947Num uniques 859; 2024-11-12T06:47:50,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=B 2024-11-12T06:47:50,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:47:50,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:47:50,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=C 2024-11-12T06:47:50,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:47:50,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/c87bd614206d491d9e2c5be89a14c3f2 is 50, key is test_row_0/A:col10/1731394069803/Put/seqid=0 2024-11-12T06:47:50,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741906_1082 (size=12301) 2024-11-12T06:47:51,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731394071009, max timestamp=1731394071025Num uniques 859; 2024-11-12T06:47:51,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T06:47:51,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731394071009, max timestamp=1731394071025Num uniques 859; 2024-11-12T06:47:51,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T06:47:51,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731394071009, max timestamp=1731394071025Num uniques 859; 2024-11-12T06:47:51,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T06:47:51,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731394071061, max timestamp=1731394071069Num uniques 1168; 2024-11-12T06:47:51,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-11-12T06:47:51,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731394071061, max timestamp=1731394071069Num uniques 1168; 2024-11-12T06:47:51,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-11-12T06:47:51,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731394071061, max timestamp=1731394071069Num uniques 1168; 2024-11-12T06:47:51,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-11-12T06:47:51,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1731394071085, max timestamp=1731394071111Num uniques 1077; 2024-11-12T06:47:51,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=17262, suffix heap size=139976,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=377 2024-11-12T06:47:51,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1731394071085, max timestamp=1731394071111Num uniques 1077; 2024-11-12T06:47:51,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1731394071085, max timestamp=1731394071111Num uniques 1077; 2024-11-12T06:47:51,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=17262, suffix heap size=139976,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=377 2024-11-12T06:47:51,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=17262, suffix heap size=139976,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=377 2024-11-12T06:47:51,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394131147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:51,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:51,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731394071144, max timestamp=1731394071158Num uniques 1236; 2024-11-12T06:47:51,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731394071144, max timestamp=1731394071158Num uniques 1236; 2024-11-12T06:47:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394131161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:51,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=56606, new segment data size=8516, suffix heap size=156512,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1236, new segment cells count=186 2024-11-12T06:47:51,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:51,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:51,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:51,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:51,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:51,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:51,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=56606, new segment data size=8516, suffix heap size=156512,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1236, new segment cells count=186 2024-11-12T06:47:51,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731394071144, max timestamp=1731394071158Num uniques 1236; 2024-11-12T06:47:51,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=56606, new segment data size=6870, suffix heap size=156512,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1236, new segment cells count=150 2024-11-12T06:47:51,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731394071194, max timestamp=1731394071204Num uniques 1045; 2024-11-12T06:47:51,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-12T06:47:51,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731394071194, max timestamp=1731394071204Num uniques 1045; 2024-11-12T06:47:51,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731394071194, max timestamp=1731394071204Num uniques 1009; 2024-11-12T06:47:51,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=11220, suffix heap size=132904,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=245 2024-11-12T06:47:51,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-12T06:47:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T06:47:51,211 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:47:51,211 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:47:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-12T06:47:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T06:47:51,214 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:47:51,215 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:47:51,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:47:51,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=354, cellSize=16210, totalHeapSize=37056, min timestamp=1731394071241, max timestamp=1731394071264Num uniques 1104; 2024-11-12T06:47:51,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=16210, suffix heap size=142784,new segment heap size=37056 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=354 2024-11-12T06:47:51,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731394071241, max timestamp=1731394071264Num uniques 1104; 2024-11-12T06:47:51,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=6870, suffix heap size=142784,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=150 2024-11-12T06:47:51,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731394071241, max timestamp=1731394071264Num uniques 1104; 2024-11-12T06:47:51,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=6870, suffix heap size=142784,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=150 2024-11-12T06:47:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T06:47:51,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1731394071295, max timestamp=1731394071357Num uniques 1009; 2024-11-12T06:47:51,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1731394071295, max timestamp=1731394071357Num uniques 1213; 2024-11-12T06:47:51,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46208, new segment data size=14328, suffix heap size=132904,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=313 2024-11-12T06:47:51,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=55548, new segment data size=14328, suffix heap size=154120,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1213, new segment cells count=313 2024-11-12T06:47:51,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731394071297, max timestamp=1731394071357Num uniques 1009; 2024-11-12T06:47:51,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46208, new segment data size=7458, suffix heap size=132904,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=163 2024-11-12T06:47:51,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T06:47:51,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:51,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:51,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:51,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:51,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:51,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/c87bd614206d491d9e2c5be89a14c3f2 2024-11-12T06:47:51,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/eb4a4e0e59134782ba5b8f3d887bf768 is 50, key is test_row_0/B:col10/1731394069803/Put/seqid=0 2024-11-12T06:47:51,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=422, cellSize=19322, totalHeapSize=44128, min timestamp=1731394071385, max timestamp=1731394071405Num uniques 1172; 2024-11-12T06:47:51,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=422, cellSize=19322, totalHeapSize=44128, min timestamp=1731394071385, max timestamp=1731394071405Num uniques 1172; 2024-11-12T06:47:51,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=19322, suffix heap size=149856,new segment heap size=44128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=422 2024-11-12T06:47:51,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=19322, suffix heap size=149856,new segment heap size=44128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=422 2024-11-12T06:47:51,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731394071392, max timestamp=1731394071408Num uniques 1022; 2024-11-12T06:47:51,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-12T06:47:51,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741907_1083 (size=12301) 2024-11-12T06:47:51,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394131436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:51,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:51,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:51,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:51,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394131437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:51,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:51,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731394071413, max timestamp=1731394071455Num uniques 1281; 2024-11-12T06:47:51,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:51,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:51,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:51,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731394071413, max timestamp=1731394071455Num uniques 1081; 2024-11-12T06:47:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394131452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:51,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731394071413, max timestamp=1731394071455Num uniques 1281; 2024-11-12T06:47:51,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=58666, new segment data size=10576, suffix heap size=161192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1281, new segment cells count=231 2024-11-12T06:47:51,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-12T06:47:51,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=58666, new segment data size=10576, suffix heap size=161192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1281, new segment cells count=231 2024-11-12T06:47:51,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:51,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:51,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:51,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T06:47:51,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:51,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:51,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:51,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:51,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T06:47:51,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731394071517, max timestamp=1731394071540Num uniques 1090; 2024-11-12T06:47:51,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731394071517, max timestamp=1731394071540Num uniques 1090; 2024-11-12T06:47:51,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-12T06:47:51,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731394071517, max timestamp=1731394071540Num uniques 1090; 2024-11-12T06:47:51,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-12T06:47:51,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-12T06:47:51,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731394071576, max timestamp=1731394071593Num uniques 1049; 2024-11-12T06:47:51,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731394071576, max timestamp=1731394071593Num uniques 1049; 2024-11-12T06:47:51,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=13694, suffix heap size=137064,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=299 2024-11-12T06:47:51,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=13694, suffix heap size=137064,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=299 2024-11-12T06:47:51,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13691, totalHeapSize=31336, min timestamp=1731394071576, max timestamp=1731394071599Num uniques 1049; 2024-11-12T06:47:51,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48041, new segment data size=13691, suffix heap size=137064,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=299 2024-11-12T06:47:51,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731394071633, max timestamp=1731394071643Num uniques 1158; 2024-11-12T06:47:51,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731394071633, max timestamp=1731394071643Num uniques 1158; 2024-11-12T06:47:51,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731394071633, max timestamp=1731394071643Num uniques 1158; 2024-11-12T06:47:51,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=9520, suffix heap size=148400,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=208 2024-11-12T06:47:51,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=9520, suffix heap size=148400,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=208 2024-11-12T06:47:51,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=9520, suffix heap size=148400,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=208 2024-11-12T06:47:51,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T06:47:51,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:51,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:51,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:51,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:51,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:51,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=417, cellSize=19092, totalHeapSize=43608, min timestamp=1731394071668, max timestamp=1731394071687Num uniques 1067; 2024-11-12T06:47:51,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=19092, suffix heap size=138936,new segment heap size=43608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=417 2024-11-12T06:47:51,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731394071668, max timestamp=1731394071687Num uniques 1067; 2024-11-12T06:47:51,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-12T06:47:51,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731394071668, max timestamp=1731394071687Num uniques 1067; 2024-11-12T06:47:51,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-12T06:47:51,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=276, cellSize=12636, totalHeapSize=28944, min timestamp=1731394071746, max timestamp=1731394071762Num uniques 1076; 2024-11-12T06:47:51,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=276, cellSize=12636, totalHeapSize=28944, min timestamp=1731394071746, max timestamp=1731394071762Num uniques 1076; 2024-11-12T06:47:51,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=276, cellSize=12636, totalHeapSize=28944, min timestamp=1731394071746, max timestamp=1731394071762Num uniques 1276; 2024-11-12T06:47:51,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=12636, suffix heap size=139872,new segment heap size=28944 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=276 2024-11-12T06:47:51,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=12636, suffix heap size=139872,new segment heap size=28944 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=276 2024-11-12T06:47:51,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=58436, new segment data size=12636, suffix heap size=160672,new segment heap size=28944 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1276, new segment cells count=276 2024-11-12T06:47:51,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10750, totalHeapSize=24680, min timestamp=1731394071796, max timestamp=1731394071807Num uniques 1135; 2024-11-12T06:47:51,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51970, new segment data size=10750, suffix heap size=146008,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1135, new segment cells count=235 2024-11-12T06:47:51,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1731394071796, max timestamp=1731394071807Num uniques 1135; 2024-11-12T06:47:51,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51980, new segment data size=8470, suffix heap size=146008,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1135, new segment cells count=185 2024-11-12T06:47:51,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1731394071796, max timestamp=1731394071807Num uniques 1135; 2024-11-12T06:47:51,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51980, new segment data size=8470, suffix heap size=146008,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1135, new segment cells count=185 2024-11-12T06:47:51,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T06:47:51,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:51,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:51,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:51,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:51,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T06:47:51,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/eb4a4e0e59134782ba5b8f3d887bf768 2024-11-12T06:47:51,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/6644c383dead41509d22ddf13051ac05 is 50, key is test_row_0/C:col10/1731394069803/Put/seqid=0 2024-11-12T06:47:51,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1731394071847, max timestamp=1731394071862Num uniques 1094; 2024-11-12T06:47:51,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=11174, suffix heap size=141744,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=244 2024-11-12T06:47:51,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1731394071847, max timestamp=1731394071862Num uniques 1044; 2024-11-12T06:47:51,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=11174, suffix heap size=136544,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=244 2024-11-12T06:47:51,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1731394071847, max timestamp=1731394071862Num uniques 1044; 2024-11-12T06:47:51,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=11174, suffix heap size=136544,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=244 2024-11-12T06:47:51,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741908_1084 (size=12301) 2024-11-12T06:47:51,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/6644c383dead41509d22ddf13051ac05 2024-11-12T06:47:51,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/c87bd614206d491d9e2c5be89a14c3f2 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/c87bd614206d491d9e2c5be89a14c3f2 2024-11-12T06:47:51,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/c87bd614206d491d9e2c5be89a14c3f2, entries=150, sequenceid=320, filesize=12.0 K 2024-11-12T06:47:51,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/eb4a4e0e59134782ba5b8f3d887bf768 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/eb4a4e0e59134782ba5b8f3d887bf768 2024-11-12T06:47:51,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/eb4a4e0e59134782ba5b8f3d887bf768, entries=150, sequenceid=320, filesize=12.0 K 2024-11-12T06:47:51,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/6644c383dead41509d22ddf13051ac05 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/6644c383dead41509d22ddf13051ac05 2024-11-12T06:47:51,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=303, cellSize=13875, totalHeapSize=31752, min timestamp=1731394071891, max timestamp=1731394071926Num uniques 1103; 2024-11-12T06:47:51,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=303, cellSize=13875, totalHeapSize=31752, min timestamp=1731394071891, max timestamp=1731394071926Num uniques 1103; 2024-11-12T06:47:51,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=13875, suffix heap size=142680,new segment heap size=31752 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=303 2024-11-12T06:47:51,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=13875, suffix heap size=142680,new segment heap size=31752 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=303 2024-11-12T06:47:51,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=303, cellSize=13875, totalHeapSize=31752, min timestamp=1731394071891, max timestamp=1731394071926Num uniques 1103; 2024-11-12T06:47:51,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=13875, suffix heap size=142680,new segment heap size=31752 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=303 2024-11-12T06:47:51,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/6644c383dead41509d22ddf13051ac05, entries=150, sequenceid=320, filesize=12.0 K 2024-11-12T06:47:51,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=60.38 KB/61830 for c0610004ec3b741dc44aa538a4d26ae4 in 989ms, sequenceid=320, compaction requested=true 2024-11-12T06:47:51,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:51,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:51,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:47:51,951 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:47:51,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:51,951 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:47:51,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:47:51,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:51,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:47:51,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:47:51,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0610004ec3b741dc44aa538a4d26ae4 3/3 column families, dataSize=87.22 KB heapSize=232.03 KB 2024-11-12T06:47:51,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=A 2024-11-12T06:47:51,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:47:51,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=B 2024-11-12T06:47:51,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:47:51,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=C 2024-11-12T06:47:51,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:47:51,954 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:47:51,955 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): c0610004ec3b741dc44aa538a4d26ae4/A is initiating minor compaction (all files) 2024-11-12T06:47:51,955 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0610004ec3b741dc44aa538a4d26ae4/A in TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:51,955 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/f77f7747c75841d3ab7c4b60b8beed4a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/5a1894ecb3234b9d9046201b66da2850, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/c87bd614206d491d9e2c5be89a14c3f2] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp, totalSize=36.7 K 2024-11-12T06:47:51,955 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:47:51,955 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): c0610004ec3b741dc44aa538a4d26ae4/B is initiating minor compaction (all files) 2024-11-12T06:47:51,955 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0610004ec3b741dc44aa538a4d26ae4/B in TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:51,955 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/c578aab538a04db59879961f84fc8e8d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/62bf03d7b0a9402fadfbd1b2a5cc7df5, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/eb4a4e0e59134782ba5b8f3d887bf768] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp, totalSize=36.7 K 2024-11-12T06:47:51,956 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting c578aab538a04db59879961f84fc8e8d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731394068562 2024-11-12T06:47:51,957 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting f77f7747c75841d3ab7c4b60b8beed4a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731394068562 2024-11-12T06:47:51,957 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 62bf03d7b0a9402fadfbd1b2a5cc7df5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731394068603 2024-11-12T06:47:51,957 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a1894ecb3234b9d9046201b66da2850, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731394068603 2024-11-12T06:47:51,957 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting eb4a4e0e59134782ba5b8f3d887bf768, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731394069800 2024-11-12T06:47:51,958 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting c87bd614206d491d9e2c5be89a14c3f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731394069800 2024-11-12T06:47:51,977 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0610004ec3b741dc44aa538a4d26ae4#A#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:47:51,978 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/981c12075308406c83ef4597a07f21f6 is 50, key is test_row_0/A:col10/1731394069803/Put/seqid=0 2024-11-12T06:47:51,978 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0610004ec3b741dc44aa538a4d26ae4#B#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:47:51,979 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/4e60fd9a02884ea186e9d076f68ecbb2 is 50, key is test_row_0/B:col10/1731394069803/Put/seqid=0 2024-11-12T06:47:51,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/ef0da875dd5d473ab35647ecd1f16cb6 is 50, key is test_row_0/A:col10/1731394071934/Put/seqid=0 2024-11-12T06:47:51,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T06:47:51,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:51,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741909_1085 (size=13119) 2024-11-12T06:47:51,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:51,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:51,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:51,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:51,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:51,996 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/981c12075308406c83ef4597a07f21f6 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/981c12075308406c83ef4597a07f21f6 2024-11-12T06:47:52,005 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0610004ec3b741dc44aa538a4d26ae4/A of c0610004ec3b741dc44aa538a4d26ae4 into 981c12075308406c83ef4597a07f21f6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:47:52,005 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:52,005 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., storeName=c0610004ec3b741dc44aa538a4d26ae4/A, priority=13, startTime=1731394071950; duration=0sec 2024-11-12T06:47:52,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394131987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,008 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:47:52,008 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:A 2024-11-12T06:47:52,009 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:47:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394131998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394132000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394132004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394132007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,011 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:47:52,012 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): c0610004ec3b741dc44aa538a4d26ae4/C is initiating minor compaction (all files) 2024-11-12T06:47:52,012 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0610004ec3b741dc44aa538a4d26ae4/C in TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:52,012 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/155c37788e224295ac60bdf46bfb6238, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/93ebed1c02184cff8f7e561f6762d966, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/6644c383dead41509d22ddf13051ac05] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp, totalSize=36.7 K 2024-11-12T06:47:52,012 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 155c37788e224295ac60bdf46bfb6238, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731394068562 2024-11-12T06:47:52,012 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93ebed1c02184cff8f7e561f6762d966, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731394068603 2024-11-12T06:47:52,013 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6644c383dead41509d22ddf13051ac05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731394069800 2024-11-12T06:47:52,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741910_1086 (size=12301) 2024-11-12T06:47:52,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=623 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/ef0da875dd5d473ab35647ecd1f16cb6 2024-11-12T06:47:52,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741911_1087 (size=13119) 2024-11-12T06:47:52,042 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0610004ec3b741dc44aa538a4d26ae4#C#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:47:52,043 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/caa3faf1f9124b23bfb2bb5f659939bb is 50, key is test_row_0/C:col10/1731394069803/Put/seqid=0 2024-11-12T06:47:52,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/867772d1eb3a409da24de82709da8159 is 50, key is test_row_0/B:col10/1731394071934/Put/seqid=0 2024-11-12T06:47:52,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741912_1088 (size=12301) 2024-11-12T06:47:52,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=623 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/867772d1eb3a409da24de82709da8159 2024-11-12T06:47:52,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741913_1089 (size=13119) 2024-11-12T06:47:52,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394132118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394132118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394132118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394132118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394132118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/6d945e322afc4755b1fde2bc6804fe4f is 50, key is test_row_0/C:col10/1731394071934/Put/seqid=0 2024-11-12T06:47:52,141 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/caa3faf1f9124b23bfb2bb5f659939bb as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/caa3faf1f9124b23bfb2bb5f659939bb 2024-11-12T06:47:52,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T06:47:52,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:52,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:52,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:52,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:52,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:52,151 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0610004ec3b741dc44aa538a4d26ae4/C of c0610004ec3b741dc44aa538a4d26ae4 into caa3faf1f9124b23bfb2bb5f659939bb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:47:52,151 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:52,151 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., storeName=c0610004ec3b741dc44aa538a4d26ae4/C, priority=13, startTime=1731394071951; duration=0sec 2024-11-12T06:47:52,151 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:52,152 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:C 2024-11-12T06:47:52,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741914_1090 (size=12301) 2024-11-12T06:47:52,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T06:47:52,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:52,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:52,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:52,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:52,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:52,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394132337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394132337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394132337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394132338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394132338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T06:47:52,441 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/4e60fd9a02884ea186e9d076f68ecbb2 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/4e60fd9a02884ea186e9d076f68ecbb2 2024-11-12T06:47:52,448 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0610004ec3b741dc44aa538a4d26ae4/B of c0610004ec3b741dc44aa538a4d26ae4 into 4e60fd9a02884ea186e9d076f68ecbb2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:47:52,448 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:52,449 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., storeName=c0610004ec3b741dc44aa538a4d26ae4/B, priority=13, startTime=1731394071951; duration=0sec 2024-11-12T06:47:52,449 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:52,449 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:B 2024-11-12T06:47:52,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T06:47:52,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:52,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:52,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:52,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:52,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:47:52,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=623 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/6d945e322afc4755b1fde2bc6804fe4f 2024-11-12T06:47:52,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/ef0da875dd5d473ab35647ecd1f16cb6 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/ef0da875dd5d473ab35647ecd1f16cb6 2024-11-12T06:47:52,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/ef0da875dd5d473ab35647ecd1f16cb6, entries=150, sequenceid=623, filesize=12.0 K 2024-11-12T06:47:52,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/867772d1eb3a409da24de82709da8159 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/867772d1eb3a409da24de82709da8159 2024-11-12T06:47:52,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/867772d1eb3a409da24de82709da8159, entries=150, sequenceid=623, filesize=12.0 K 2024-11-12T06:47:52,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/6d945e322afc4755b1fde2bc6804fe4f as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/6d945e322afc4755b1fde2bc6804fe4f 2024-11-12T06:47:52,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/6d945e322afc4755b1fde2bc6804fe4f, entries=150, sequenceid=623, filesize=12.0 K 2024-11-12T06:47:52,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~251.91 KB/257952, currentSize=93.93 KB/96180 for c0610004ec3b741dc44aa538a4d26ae4 in 631ms, sequenceid=623, compaction requested=false 2024-11-12T06:47:52,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:52,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T06:47:52,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:52,604 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing c0610004ec3b741dc44aa538a4d26ae4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T06:47:52,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=A 2024-11-12T06:47:52,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:52,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=B 2024-11-12T06:47:52,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:52,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=C 2024-11-12T06:47:52,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:52,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/0bf5a8eb4dc54c5d9e8be79275406006 is 50, key is test_row_0/A:col10/1731394071995/Put/seqid=0 2024-11-12T06:47:52,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741915_1091 (size=12301) 2024-11-12T06:47:52,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. as already flushing 2024-11-12T06:47:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:52,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394132657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394132658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394132659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394132660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394132665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394132766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394132777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394132777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394132777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394132777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394132977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394132987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394132987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394132987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394132987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:52,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:52,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:53,014 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=643 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/0bf5a8eb4dc54c5d9e8be79275406006 2024-11-12T06:47:53,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/d60555cde0834faa8189745630ee69a4 is 50, key is test_row_0/B:col10/1731394071995/Put/seqid=0 2024-11-12T06:47:53,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741916_1092 (size=12301) 2024-11-12T06:47:53,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:53,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394133287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:53,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:53,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394133297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:53,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:53,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:53,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:53,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:53,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394133297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:53,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:53,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:53,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:53,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:53,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394133297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:53,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:53,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:53,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:53,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:53,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394133297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:53,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:53,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:53,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T06:47:53,473 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=643 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/d60555cde0834faa8189745630ee69a4 2024-11-12T06:47:53,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/8720add442554f2eac3a1a799eb4293b is 50, key is test_row_0/C:col10/1731394071995/Put/seqid=0 2024-11-12T06:47:53,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741917_1093 (size=12301) 2024-11-12T06:47:53,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43360 deadline: 1731394133797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:53,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:53,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43318 deadline: 1731394133807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:53,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:53,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:53,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:53,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43370 deadline: 1731394133807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:53,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:53,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:53,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:53,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43400 deadline: 1731394133807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:53,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:53,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:53,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:53,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:47:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43392 deadline: 1731394133817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:53,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:53,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:47:53,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:47:53,887 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=643 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/8720add442554f2eac3a1a799eb4293b 2024-11-12T06:47:53,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/0bf5a8eb4dc54c5d9e8be79275406006 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/0bf5a8eb4dc54c5d9e8be79275406006 2024-11-12T06:47:53,898 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/0bf5a8eb4dc54c5d9e8be79275406006, entries=150, sequenceid=643, filesize=12.0 K 2024-11-12T06:47:53,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/d60555cde0834faa8189745630ee69a4 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/d60555cde0834faa8189745630ee69a4 2024-11-12T06:47:53,904 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/d60555cde0834faa8189745630ee69a4, entries=150, sequenceid=643, filesize=12.0 K 2024-11-12T06:47:53,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/8720add442554f2eac3a1a799eb4293b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/8720add442554f2eac3a1a799eb4293b 2024-11-12T06:47:53,912 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/8720add442554f2eac3a1a799eb4293b, entries=150, sequenceid=643, filesize=12.0 K 2024-11-12T06:47:53,914 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c0610004ec3b741dc44aa538a4d26ae4 in 1309ms, sequenceid=643, compaction requested=true 2024-11-12T06:47:53,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:53,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:53,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-12T06:47:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-12T06:47:53,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-12T06:47:53,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6990 sec 2024-11-12T06:47:53,920 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.7070 sec 2024-11-12T06:47:54,142 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-12T06:47:54,142 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:47:54,144 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-12T06:47:54,144 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:47:54,145 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-12T06:47:54,145 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:47:54,147 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:54,147 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:54,147 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:54,147 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-12T06:47:54,147 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:47:54,147 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:54,147 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:54,147 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:54,147 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:47:54,147 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:47:54,147 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:54,148 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:47:54,148 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:54,148 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:47:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:54,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0610004ec3b741dc44aa538a4d26ae4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T06:47:54,814 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-12T06:47:54,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=A 2024-11-12T06:47:54,814 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:47:54,814 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:54,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:54,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=B 2024-11-12T06:47:54,815 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:54,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:54,815 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:47:54,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=C 2024-11-12T06:47:54,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:54,820 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-12T06:47:54,820 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-12T06:47:54,820 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:47:54,820 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:47:54,820 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:54,820 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:54,820 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:54,820 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:47:54,820 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:54,820 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:47:54,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/1dbc9d60800146e1988e56c283b73684 is 50, key is test_row_0/A:col10/1731394072656/Put/seqid=0 2024-11-12T06:47:54,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741918_1094 (size=12301) 2024-11-12T06:47:54,828 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-12T06:47:54,828 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-12T06:47:54,828 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:47:54,828 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:47:54,829 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:54,829 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:54,829 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:54,829 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:54,829 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:47:54,829 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:47:55,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=659 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/1dbc9d60800146e1988e56c283b73684 2024-11-12T06:47:55,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/008537a49af44780a7738cc3eb729d7f is 50, key is test_row_0/B:col10/1731394072656/Put/seqid=0 2024-11-12T06:47:55,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741919_1095 (size=12301) 2024-11-12T06:47:55,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T06:47:55,389 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:47:55,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T06:47:55,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-11-12T06:47:55,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-12T06:47:55,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 120 2024-11-12T06:47:55,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 120 2024-11-12T06:47:55,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-11-12T06:47:55,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T06:47:55,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2771 2024-11-12T06:47:55,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2711 2024-11-12T06:47:55,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T06:47:55,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1219 2024-11-12T06:47:55,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3650 rows 2024-11-12T06:47:55,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1225 2024-11-12T06:47:55,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3671 rows 2024-11-12T06:47:55,393 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T06:47:55,393 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T06:47:55,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:55,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:55,394 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:47:55,399 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T06:47:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T06:47:55,407 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394075407"}]},"ts":"1731394075407"} 2024-11-12T06:47:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T06:47:55,409 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T06:47:55,409 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T06:47:55,411 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T06:47:55,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0610004ec3b741dc44aa538a4d26ae4, UNASSIGN}] 2024-11-12T06:47:55,416 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0610004ec3b741dc44aa538a4d26ae4, UNASSIGN 2024-11-12T06:47:55,417 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=c0610004ec3b741dc44aa538a4d26ae4, regionState=CLOSING, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:55,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0610004ec3b741dc44aa538a4d26ae4, UNASSIGN because future has completed 2024-11-12T06:47:55,420 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T06:47:55,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622}] 2024-11-12T06:47:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T06:47:55,579 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:55,579 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T06:47:55,581 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing c0610004ec3b741dc44aa538a4d26ae4, disabling compactions & flushes 2024-11-12T06:47:55,581 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:55,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=659 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/008537a49af44780a7738cc3eb729d7f 2024-11-12T06:47:55,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/3219d63976f04fe39c05305ef20e2202 is 50, key is test_row_0/C:col10/1731394072656/Put/seqid=0 2024-11-12T06:47:55,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741920_1096 (size=12301) 2024-11-12T06:47:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T06:47:55,968 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T06:47:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T06:47:56,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=659 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/3219d63976f04fe39c05305ef20e2202 2024-11-12T06:47:56,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/1dbc9d60800146e1988e56c283b73684 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/1dbc9d60800146e1988e56c283b73684 2024-11-12T06:47:56,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/1dbc9d60800146e1988e56c283b73684, entries=150, sequenceid=659, filesize=12.0 K 2024-11-12T06:47:56,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/008537a49af44780a7738cc3eb729d7f as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/008537a49af44780a7738cc3eb729d7f 2024-11-12T06:47:56,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/008537a49af44780a7738cc3eb729d7f, entries=150, sequenceid=659, filesize=12.0 K 2024-11-12T06:47:56,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/3219d63976f04fe39c05305ef20e2202 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/3219d63976f04fe39c05305ef20e2202 2024-11-12T06:47:56,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/3219d63976f04fe39c05305ef20e2202, entries=150, sequenceid=659, filesize=12.0 K 2024-11-12T06:47:56,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=26.84 KB/27480 for c0610004ec3b741dc44aa538a4d26ae4 in 1285ms, sequenceid=659, compaction requested=true 2024-11-12T06:47:56,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0610004ec3b741dc44aa538a4d26ae4: 2024-11-12T06:47:56,098 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:56,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:47:56,098 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:56,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:56,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:47:56,098 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. after waiting 0 ms 2024-11-12T06:47:56,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:47:56,098 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. because compaction request was cancelled 2024-11-12T06:47:56,098 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. because compaction request was cancelled 2024-11-12T06:47:56,098 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:56,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0610004ec3b741dc44aa538a4d26ae4:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:47:56,098 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:B 2024-11-12T06:47:56,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:47:56,098 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:A 2024-11-12T06:47:56,098 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. because compaction request was cancelled 2024-11-12T06:47:56,099 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0610004ec3b741dc44aa538a4d26ae4:C 2024-11-12T06:47:56,099 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing c0610004ec3b741dc44aa538a4d26ae4 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-12T06:47:56,099 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=A 2024-11-12T06:47:56,099 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:56,099 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=B 2024-11-12T06:47:56,099 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:56,099 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0610004ec3b741dc44aa538a4d26ae4, store=C 2024-11-12T06:47:56,099 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:47:56,103 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/d631f760f7e049a7ad85e5e703ae7cf9 is 50, key is test_row_1/A:col10/1731394074827/Put/seqid=0 2024-11-12T06:47:56,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741921_1097 (size=7415) 2024-11-12T06:47:56,508 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=666 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/d631f760f7e049a7ad85e5e703ae7cf9 2024-11-12T06:47:56,523 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/d7261d269fc34af5bda7d1825d661791 is 50, key is test_row_1/B:col10/1731394074827/Put/seqid=0 2024-11-12T06:47:56,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741922_1098 (size=7415) 2024-11-12T06:47:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T06:47:56,927 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=666 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/d7261d269fc34af5bda7d1825d661791 2024-11-12T06:47:56,936 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/698edf126906451b949b7d9a1ce4402c is 50, key is test_row_1/C:col10/1731394074827/Put/seqid=0 2024-11-12T06:47:56,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741923_1099 (size=7415) 2024-11-12T06:47:57,341 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=666 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/698edf126906451b949b7d9a1ce4402c 2024-11-12T06:47:57,348 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/A/d631f760f7e049a7ad85e5e703ae7cf9 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/d631f760f7e049a7ad85e5e703ae7cf9 2024-11-12T06:47:57,353 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/d631f760f7e049a7ad85e5e703ae7cf9, entries=50, sequenceid=666, filesize=7.2 K 2024-11-12T06:47:57,354 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/B/d7261d269fc34af5bda7d1825d661791 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/d7261d269fc34af5bda7d1825d661791 2024-11-12T06:47:57,360 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/d7261d269fc34af5bda7d1825d661791, entries=50, sequenceid=666, filesize=7.2 K 2024-11-12T06:47:57,361 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/.tmp/C/698edf126906451b949b7d9a1ce4402c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/698edf126906451b949b7d9a1ce4402c 2024-11-12T06:47:57,366 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/698edf126906451b949b7d9a1ce4402c, entries=50, sequenceid=666, filesize=7.2 K 2024-11-12T06:47:57,367 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c0610004ec3b741dc44aa538a4d26ae4 in 1269ms, sequenceid=666, compaction requested=true 2024-11-12T06:47:57,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/b9d0ef4dc5f94173a0ac05b506665759, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/0764c294fe0043e4bccde82a3f7aab12, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/9806d410050b4270893e4142c22be671, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/fbe03c95159248e28c47d37e72dd6b76, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/89cd9e0803c24edcb049ca51532bc467, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/67ed5cee0f534a88bbdd44d7136c0ddd, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/3befc5db9ca94729bc8c0c9704b4d1de, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/30a8ad65d21a412385164d71c7fa7949, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/dbf076f09c3c413e8a77d9610734393d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/386a11924be94287ae0be47c768a7789, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/706aa1b9770f43388edfeb0c799e138f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/87bfcca0b3524167bcde3742563f15c2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/f4e9c9e3ae0143b790d293827ede5283, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/8f1d1bf68e0d462b8a87df47ae2ffacc, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/c2be7ade369948369ac90b320676452e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/e7c37cb8fcd944428976c225898d4f06, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/a6d90d23ad85496c90190cd3c6b4915b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/e0bbbf8995fd408fbb02ec20ec1e6ec5, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/33412a58247c4612a6f32195dde3cf5b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/cf8a5358f5864b9d9335cd7a54e34faa, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/f77f7747c75841d3ab7c4b60b8beed4a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/b0dedabb9478428195d054a595e4532b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/5a1894ecb3234b9d9046201b66da2850, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/c87bd614206d491d9e2c5be89a14c3f2] to archive 2024-11-12T06:47:57,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T06:47:57,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/b9d0ef4dc5f94173a0ac05b506665759 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/b9d0ef4dc5f94173a0ac05b506665759 2024-11-12T06:47:57,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/0764c294fe0043e4bccde82a3f7aab12 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/0764c294fe0043e4bccde82a3f7aab12 2024-11-12T06:47:57,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/9806d410050b4270893e4142c22be671 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/9806d410050b4270893e4142c22be671 2024-11-12T06:47:57,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/fbe03c95159248e28c47d37e72dd6b76 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/fbe03c95159248e28c47d37e72dd6b76 2024-11-12T06:47:57,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/89cd9e0803c24edcb049ca51532bc467 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/89cd9e0803c24edcb049ca51532bc467 2024-11-12T06:47:57,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/67ed5cee0f534a88bbdd44d7136c0ddd to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/67ed5cee0f534a88bbdd44d7136c0ddd 2024-11-12T06:47:57,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/3befc5db9ca94729bc8c0c9704b4d1de to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/3befc5db9ca94729bc8c0c9704b4d1de 2024-11-12T06:47:57,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/30a8ad65d21a412385164d71c7fa7949 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/30a8ad65d21a412385164d71c7fa7949 2024-11-12T06:47:57,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/dbf076f09c3c413e8a77d9610734393d to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/dbf076f09c3c413e8a77d9610734393d 2024-11-12T06:47:57,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/386a11924be94287ae0be47c768a7789 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/386a11924be94287ae0be47c768a7789 2024-11-12T06:47:57,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/706aa1b9770f43388edfeb0c799e138f to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/706aa1b9770f43388edfeb0c799e138f 2024-11-12T06:47:57,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/87bfcca0b3524167bcde3742563f15c2 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/87bfcca0b3524167bcde3742563f15c2 2024-11-12T06:47:57,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/f4e9c9e3ae0143b790d293827ede5283 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/f4e9c9e3ae0143b790d293827ede5283 2024-11-12T06:47:57,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/8f1d1bf68e0d462b8a87df47ae2ffacc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/8f1d1bf68e0d462b8a87df47ae2ffacc 2024-11-12T06:47:57,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/c2be7ade369948369ac90b320676452e to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/c2be7ade369948369ac90b320676452e 2024-11-12T06:47:57,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/e7c37cb8fcd944428976c225898d4f06 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/e7c37cb8fcd944428976c225898d4f06 2024-11-12T06:47:57,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/a6d90d23ad85496c90190cd3c6b4915b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/a6d90d23ad85496c90190cd3c6b4915b 2024-11-12T06:47:57,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/e0bbbf8995fd408fbb02ec20ec1e6ec5 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/e0bbbf8995fd408fbb02ec20ec1e6ec5 2024-11-12T06:47:57,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/33412a58247c4612a6f32195dde3cf5b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/33412a58247c4612a6f32195dde3cf5b 2024-11-12T06:47:57,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/cf8a5358f5864b9d9335cd7a54e34faa to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/cf8a5358f5864b9d9335cd7a54e34faa 2024-11-12T06:47:57,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/f77f7747c75841d3ab7c4b60b8beed4a to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/f77f7747c75841d3ab7c4b60b8beed4a 2024-11-12T06:47:57,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/b0dedabb9478428195d054a595e4532b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/b0dedabb9478428195d054a595e4532b 2024-11-12T06:47:57,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/5a1894ecb3234b9d9046201b66da2850 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/5a1894ecb3234b9d9046201b66da2850 2024-11-12T06:47:57,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/c87bd614206d491d9e2c5be89a14c3f2 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/c87bd614206d491d9e2c5be89a14c3f2 2024-11-12T06:47:57,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/fe892627c554415283db3b84b5fe1415, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/fdecbc143aed4323bb577f10b0e6e5a1, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/74458a834c6d46a5892ae82385cc3b0c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/8da8c0ec2d8d4431868033152745cad0, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/f2f5fd3d30e2407aafaac5e1ae606399, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/9af5abf59b704ae5b0f080b92f5b2bf5, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/ea9bd9bf0fa64ee4a6bb04cf7f018b9e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/00a381ca78ba42248d3b21cffd947c3c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/17ad220ddddf4ab08ba80b582323fb90, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/144146f8fd2546a2859fe35bc0608c8a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/8e7e4041ca364591b74bccd851b36e65, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/d41697b7659d4166a1a6f74543592805, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/478ebae8b5434a5f8e49fd3ad58b399b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/825c59b4ad974881b11f37894cbe9084, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/442f39eb7c84495eb7b8e33f1ac0ff1e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/07ecae6e10b34f6c9e8ab63b3afc0f2d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/7d4c3bf5a71b4e27a57df1829f26a348, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/0da492c92db34c799346be197b4dc8b8, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/805fc7f0b9c24cb091e8431c9a3c2ced, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/f88c014154cd449d92566d5585f07e2f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/c578aab538a04db59879961f84fc8e8d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/b56628d53e9e41129ea27d34d63f5ef1, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/62bf03d7b0a9402fadfbd1b2a5cc7df5, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/eb4a4e0e59134782ba5b8f3d887bf768] to archive 2024-11-12T06:47:57,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T06:47:57,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/fe892627c554415283db3b84b5fe1415 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/fe892627c554415283db3b84b5fe1415 2024-11-12T06:47:57,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/fdecbc143aed4323bb577f10b0e6e5a1 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/fdecbc143aed4323bb577f10b0e6e5a1 2024-11-12T06:47:57,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/74458a834c6d46a5892ae82385cc3b0c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/74458a834c6d46a5892ae82385cc3b0c 2024-11-12T06:47:57,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/8da8c0ec2d8d4431868033152745cad0 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/8da8c0ec2d8d4431868033152745cad0 2024-11-12T06:47:57,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/f2f5fd3d30e2407aafaac5e1ae606399 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/f2f5fd3d30e2407aafaac5e1ae606399 2024-11-12T06:47:57,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/9af5abf59b704ae5b0f080b92f5b2bf5 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/9af5abf59b704ae5b0f080b92f5b2bf5 2024-11-12T06:47:57,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/ea9bd9bf0fa64ee4a6bb04cf7f018b9e to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/ea9bd9bf0fa64ee4a6bb04cf7f018b9e 2024-11-12T06:47:57,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/00a381ca78ba42248d3b21cffd947c3c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/00a381ca78ba42248d3b21cffd947c3c 2024-11-12T06:47:57,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/17ad220ddddf4ab08ba80b582323fb90 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/17ad220ddddf4ab08ba80b582323fb90 2024-11-12T06:47:57,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/144146f8fd2546a2859fe35bc0608c8a to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/144146f8fd2546a2859fe35bc0608c8a 2024-11-12T06:47:57,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/8e7e4041ca364591b74bccd851b36e65 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/8e7e4041ca364591b74bccd851b36e65 2024-11-12T06:47:57,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/d41697b7659d4166a1a6f74543592805 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/d41697b7659d4166a1a6f74543592805 2024-11-12T06:47:57,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/478ebae8b5434a5f8e49fd3ad58b399b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/478ebae8b5434a5f8e49fd3ad58b399b 2024-11-12T06:47:57,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/825c59b4ad974881b11f37894cbe9084 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/825c59b4ad974881b11f37894cbe9084 2024-11-12T06:47:57,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/442f39eb7c84495eb7b8e33f1ac0ff1e to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/442f39eb7c84495eb7b8e33f1ac0ff1e 2024-11-12T06:47:57,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/07ecae6e10b34f6c9e8ab63b3afc0f2d to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/07ecae6e10b34f6c9e8ab63b3afc0f2d 2024-11-12T06:47:57,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/7d4c3bf5a71b4e27a57df1829f26a348 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/7d4c3bf5a71b4e27a57df1829f26a348 2024-11-12T06:47:57,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/0da492c92db34c799346be197b4dc8b8 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/0da492c92db34c799346be197b4dc8b8 2024-11-12T06:47:57,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/805fc7f0b9c24cb091e8431c9a3c2ced to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/805fc7f0b9c24cb091e8431c9a3c2ced 2024-11-12T06:47:57,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/f88c014154cd449d92566d5585f07e2f to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/f88c014154cd449d92566d5585f07e2f 2024-11-12T06:47:57,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/c578aab538a04db59879961f84fc8e8d to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/c578aab538a04db59879961f84fc8e8d 2024-11-12T06:47:57,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/b56628d53e9e41129ea27d34d63f5ef1 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/b56628d53e9e41129ea27d34d63f5ef1 2024-11-12T06:47:57,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/62bf03d7b0a9402fadfbd1b2a5cc7df5 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/62bf03d7b0a9402fadfbd1b2a5cc7df5 2024-11-12T06:47:57,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/eb4a4e0e59134782ba5b8f3d887bf768 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/eb4a4e0e59134782ba5b8f3d887bf768 2024-11-12T06:47:57,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/2511efec87894c45a30f8bff4941c220, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/1c3a323be5954a9eaee2396bb1747128, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/036916536e1d45ffbd2713d04c02b792, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/c612fb00ae65436892c95b95a37af328, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/3355ba47e6824dccb92f98b4e6a32fea, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/42a06668792c495294f92006a2d5d8c3, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/e3484f0bc53141208b0e76829d07a334, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/8972c365046b447cbbcf9cca9830f047, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/95597434f0f74640b70a1c3df5aa7ac1, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/1037b62e62a24375ae0682e2dbdc33fd, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/a30409c93d304621a27b720faa112a5c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/db53880fe2a2426194fd31f1ae0e9fed, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/ab1eee51794e49a58f555c8794ff83b8, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/e4a872d499714e54b160aed811158bd8, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/c04778e836b24612a1a983a88c9b9668, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/a6a0a2c75760452eadeefcb89ed18aac, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/30aa4aceaf954859b3209a9424dc23d3, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/7a6bd2b0d63f4040a7b9d73bc1ddfc81, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/9f3a449b41d64f5d8d713fc398f845ff, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/11714193cd3a428484336554bd009e0b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/155c37788e224295ac60bdf46bfb6238, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/6325cf4c39884dbeba4d8be320d0c614, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/93ebed1c02184cff8f7e561f6762d966, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/6644c383dead41509d22ddf13051ac05] to archive 2024-11-12T06:47:57,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T06:47:57,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/2511efec87894c45a30f8bff4941c220 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/2511efec87894c45a30f8bff4941c220 2024-11-12T06:47:57,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/1c3a323be5954a9eaee2396bb1747128 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/1c3a323be5954a9eaee2396bb1747128 2024-11-12T06:47:57,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/036916536e1d45ffbd2713d04c02b792 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/036916536e1d45ffbd2713d04c02b792 2024-11-12T06:47:57,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/c612fb00ae65436892c95b95a37af328 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/c612fb00ae65436892c95b95a37af328 2024-11-12T06:47:57,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/3355ba47e6824dccb92f98b4e6a32fea to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/3355ba47e6824dccb92f98b4e6a32fea 2024-11-12T06:47:57,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/42a06668792c495294f92006a2d5d8c3 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/42a06668792c495294f92006a2d5d8c3 2024-11-12T06:47:57,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/e3484f0bc53141208b0e76829d07a334 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/e3484f0bc53141208b0e76829d07a334 2024-11-12T06:47:57,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/8972c365046b447cbbcf9cca9830f047 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/8972c365046b447cbbcf9cca9830f047 2024-11-12T06:47:57,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/95597434f0f74640b70a1c3df5aa7ac1 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/95597434f0f74640b70a1c3df5aa7ac1 2024-11-12T06:47:57,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/1037b62e62a24375ae0682e2dbdc33fd to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/1037b62e62a24375ae0682e2dbdc33fd 2024-11-12T06:47:57,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/a30409c93d304621a27b720faa112a5c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/a30409c93d304621a27b720faa112a5c 2024-11-12T06:47:57,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/db53880fe2a2426194fd31f1ae0e9fed to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/db53880fe2a2426194fd31f1ae0e9fed 2024-11-12T06:47:57,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/ab1eee51794e49a58f555c8794ff83b8 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/ab1eee51794e49a58f555c8794ff83b8 2024-11-12T06:47:57,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/e4a872d499714e54b160aed811158bd8 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/e4a872d499714e54b160aed811158bd8 2024-11-12T06:47:57,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/c04778e836b24612a1a983a88c9b9668 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/c04778e836b24612a1a983a88c9b9668 2024-11-12T06:47:57,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/a6a0a2c75760452eadeefcb89ed18aac to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/a6a0a2c75760452eadeefcb89ed18aac 2024-11-12T06:47:57,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/30aa4aceaf954859b3209a9424dc23d3 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/30aa4aceaf954859b3209a9424dc23d3 2024-11-12T06:47:57,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/7a6bd2b0d63f4040a7b9d73bc1ddfc81 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/7a6bd2b0d63f4040a7b9d73bc1ddfc81 2024-11-12T06:47:57,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/9f3a449b41d64f5d8d713fc398f845ff to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/9f3a449b41d64f5d8d713fc398f845ff 2024-11-12T06:47:57,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/11714193cd3a428484336554bd009e0b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/11714193cd3a428484336554bd009e0b 2024-11-12T06:47:57,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/155c37788e224295ac60bdf46bfb6238 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/155c37788e224295ac60bdf46bfb6238 2024-11-12T06:47:57,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/6325cf4c39884dbeba4d8be320d0c614 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/6325cf4c39884dbeba4d8be320d0c614 2024-11-12T06:47:57,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/93ebed1c02184cff8f7e561f6762d966 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/93ebed1c02184cff8f7e561f6762d966 2024-11-12T06:47:57,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/6644c383dead41509d22ddf13051ac05 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/6644c383dead41509d22ddf13051ac05 2024-11-12T06:47:57,483 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/recovered.edits/669.seqid, newMaxSeqId=669, maxSeqId=1 2024-11-12T06:47:57,483 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. 2024-11-12T06:47:57,484 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for c0610004ec3b741dc44aa538a4d26ae4: Waiting for close lock at 1731394075581Running coprocessor pre-close hooks at 1731394075581Disabling compacts and flushes for region at 1731394075581Disabling writes for close at 1731394076098 (+517 ms)Obtaining lock to block concurrent updates at 1731394076099 (+1 ms)Preparing flush snapshotting stores in c0610004ec3b741dc44aa538a4d26ae4 at 1731394076099Finished memstore snapshotting TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731394076099Flushing stores of TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4. at 1731394076100 (+1 ms)Flushing c0610004ec3b741dc44aa538a4d26ae4/A: creating writer at 1731394076100Flushing c0610004ec3b741dc44aa538a4d26ae4/A: appending metadata at 1731394076102 (+2 ms)Flushing c0610004ec3b741dc44aa538a4d26ae4/A: closing flushed file at 1731394076102Flushing c0610004ec3b741dc44aa538a4d26ae4/B: creating writer at 1731394076519 (+417 ms)Flushing c0610004ec3b741dc44aa538a4d26ae4/B: appending metadata at 1731394076522 (+3 ms)Flushing c0610004ec3b741dc44aa538a4d26ae4/B: closing flushed file at 1731394076522Flushing c0610004ec3b741dc44aa538a4d26ae4/C: creating writer at 1731394076933 (+411 ms)Flushing c0610004ec3b741dc44aa538a4d26ae4/C: appending metadata at 1731394076936 (+3 ms)Flushing c0610004ec3b741dc44aa538a4d26ae4/C: closing flushed file at 1731394076936Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@772bc952: reopening flushed file at 1731394077347 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c6ecd83: reopening flushed file at 1731394077353 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a4a868d: reopening flushed file at 1731394077360 (+7 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c0610004ec3b741dc44aa538a4d26ae4 in 1269ms, sequenceid=666, compaction requested=true at 1731394077367 (+7 ms)Writing region close event to WAL at 1731394077479 (+112 ms)Running coprocessor post-close hooks at 1731394077483 (+4 ms)Closed at 1731394077483 2024-11-12T06:47:57,486 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:57,486 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=c0610004ec3b741dc44aa538a4d26ae4, regionState=CLOSED 2024-11-12T06:47:57,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 because future has completed 2024-11-12T06:47:57,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-12T06:47:57,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure c0610004ec3b741dc44aa538a4d26ae4, server=9f0beee3d6b6,38777,1731394047622 in 2.0700 sec 2024-11-12T06:47:57,494 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-12T06:47:57,494 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0610004ec3b741dc44aa538a4d26ae4, UNASSIGN in 2.0770 sec 2024-11-12T06:47:57,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-12T06:47:57,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0850 sec 2024-11-12T06:47:57,499 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394077499"}]},"ts":"1731394077499"} 2024-11-12T06:47:57,501 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T06:47:57,501 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T06:47:57,503 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1000 sec 2024-11-12T06:47:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T06:47:57,558 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:47:57,562 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T06:47:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:47:57,569 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:47:57,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-12T06:47:57,572 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:47:57,579 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:57,583 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A, FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B, FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C, FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/recovered.edits] 2024-11-12T06:47:57,587 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/0bf5a8eb4dc54c5d9e8be79275406006 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/0bf5a8eb4dc54c5d9e8be79275406006 2024-11-12T06:47:57,588 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/1dbc9d60800146e1988e56c283b73684 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/1dbc9d60800146e1988e56c283b73684 2024-11-12T06:47:57,590 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/981c12075308406c83ef4597a07f21f6 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/981c12075308406c83ef4597a07f21f6 2024-11-12T06:47:57,591 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/d631f760f7e049a7ad85e5e703ae7cf9 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/d631f760f7e049a7ad85e5e703ae7cf9 2024-11-12T06:47:57,592 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/ef0da875dd5d473ab35647ecd1f16cb6 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/A/ef0da875dd5d473ab35647ecd1f16cb6 2024-11-12T06:47:57,595 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/008537a49af44780a7738cc3eb729d7f to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/008537a49af44780a7738cc3eb729d7f 2024-11-12T06:47:57,596 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/4e60fd9a02884ea186e9d076f68ecbb2 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/4e60fd9a02884ea186e9d076f68ecbb2 2024-11-12T06:47:57,597 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/867772d1eb3a409da24de82709da8159 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/867772d1eb3a409da24de82709da8159 2024-11-12T06:47:57,598 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/d60555cde0834faa8189745630ee69a4 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/d60555cde0834faa8189745630ee69a4 2024-11-12T06:47:57,600 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/d7261d269fc34af5bda7d1825d661791 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/B/d7261d269fc34af5bda7d1825d661791 2024-11-12T06:47:57,602 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/3219d63976f04fe39c05305ef20e2202 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/3219d63976f04fe39c05305ef20e2202 2024-11-12T06:47:57,603 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/698edf126906451b949b7d9a1ce4402c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/698edf126906451b949b7d9a1ce4402c 2024-11-12T06:47:57,605 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/6d945e322afc4755b1fde2bc6804fe4f to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/6d945e322afc4755b1fde2bc6804fe4f 2024-11-12T06:47:57,606 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/8720add442554f2eac3a1a799eb4293b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/8720add442554f2eac3a1a799eb4293b 2024-11-12T06:47:57,607 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/caa3faf1f9124b23bfb2bb5f659939bb to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/C/caa3faf1f9124b23bfb2bb5f659939bb 2024-11-12T06:47:57,610 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/recovered.edits/669.seqid to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4/recovered.edits/669.seqid 2024-11-12T06:47:57,611 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/c0610004ec3b741dc44aa538a4d26ae4 2024-11-12T06:47:57,611 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T06:47:57,616 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:47:57,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-12T06:47:57,628 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T06:47:57,656 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T06:47:57,658 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:47:57,659 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T06:47:57,659 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731394077659"}]},"ts":"9223372036854775807"} 2024-11-12T06:47:57,664 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T06:47:57,664 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c0610004ec3b741dc44aa538a4d26ae4, NAME => 'TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T06:47:57,664 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T06:47:57,665 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731394077664"}]},"ts":"9223372036854775807"} 2024-11-12T06:47:57,669 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T06:47:57,670 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:47:57,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 107 msec 2024-11-12T06:47:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-12T06:47:57,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T06:47:57,679 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:47:57,691 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;9f0beee3d6b6:38777-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_455841092_22 at /127.0.0.1:45420 [Waiting for operation #262] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-553601269_22 at /127.0.0.1:39144 [Waiting for operation #278] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=273 (was 135) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6914 (was 7499) 2024-11-12T06:47:57,700 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=273, ProcessCount=11, AvailableMemoryMB=6914 2024-11-12T06:47:57,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T06:47:57,702 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T06:47:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T06:47:57,704 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T06:47:57,704 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:47:57,704 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-12T06:47:57,705 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T06:47:57,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-12T06:47:57,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741924_1100 (size=1061) 2024-11-12T06:47:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-12T06:47:58,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-12T06:47:58,114 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e 2024-11-12T06:47:58,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741925_1101 (size=53) 2024-11-12T06:47:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-12T06:47:58,525 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T06:47:58,525 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0a0c05661b31342e3af97a578b0b00fc, disabling compactions & flushes 2024-11-12T06:47:58,525 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:47:58,526 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:47:58,526 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. after waiting 0 ms 2024-11-12T06:47:58,526 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:47:58,526 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:47:58,526 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0a0c05661b31342e3af97a578b0b00fc: Waiting for close lock at 1731394078525Disabling compacts and flushes for region at 1731394078525Disabling writes for close at 1731394078526 (+1 ms)Writing region close event to WAL at 1731394078526Closed at 1731394078526 2024-11-12T06:47:58,530 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T06:47:58,531 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731394078530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731394078530"}]},"ts":"1731394078530"} 2024-11-12T06:47:58,534 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T06:47:58,536 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T06:47:58,536 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394078536"}]},"ts":"1731394078536"} 2024-11-12T06:47:58,538 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T06:47:58,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0a0c05661b31342e3af97a578b0b00fc, ASSIGN}] 2024-11-12T06:47:58,539 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0a0c05661b31342e3af97a578b0b00fc, ASSIGN 2024-11-12T06:47:58,540 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0a0c05661b31342e3af97a578b0b00fc, ASSIGN; state=OFFLINE, location=9f0beee3d6b6,38777,1731394047622; forceNewPlan=false, retain=false 2024-11-12T06:47:58,691 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=0a0c05661b31342e3af97a578b0b00fc, regionState=OPENING, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:58,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0a0c05661b31342e3af97a578b0b00fc, ASSIGN because future has completed 2024-11-12T06:47:58,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622}] 2024-11-12T06:47:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-12T06:47:58,853 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:47:58,854 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} 2024-11-12T06:47:58,855 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:47:58,855 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T06:47:58,855 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:47:58,855 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:47:58,857 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:47:58,858 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:47:58,859 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a0c05661b31342e3af97a578b0b00fc columnFamilyName A 2024-11-12T06:47:58,859 DEBUG [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:47:58,860 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] regionserver.HStore(327): Store=0a0c05661b31342e3af97a578b0b00fc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:47:58,860 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:47:58,861 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:47:58,861 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a0c05661b31342e3af97a578b0b00fc columnFamilyName B 2024-11-12T06:47:58,861 DEBUG [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:47:58,862 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] regionserver.HStore(327): Store=0a0c05661b31342e3af97a578b0b00fc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:47:58,862 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:47:58,863 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:47:58,863 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a0c05661b31342e3af97a578b0b00fc columnFamilyName C 2024-11-12T06:47:58,863 DEBUG [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:47:58,864 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] regionserver.HStore(327): Store=0a0c05661b31342e3af97a578b0b00fc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:47:58,864 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:47:58,864 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:47:58,865 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:47:58,865 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:47:58,866 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:47:58,866 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:47:58,867 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T06:47:58,868 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:47:58,870 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T06:47:58,871 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 0a0c05661b31342e3af97a578b0b00fc; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64343972, jitterRate=-0.041200101375579834}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T06:47:58,871 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:47:58,872 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 0a0c05661b31342e3af97a578b0b00fc: Running coprocessor pre-open hook at 1731394078856Writing region info on filesystem at 1731394078856Initializing all the Stores at 1731394078857 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394078857Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394078857Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394078857Cleaning up temporary data from old regions at 1731394078866 (+9 ms)Running coprocessor post-open hooks at 1731394078871 (+5 ms)Region opened successfully at 1731394078872 (+1 ms) 2024-11-12T06:47:58,872 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., pid=30, masterSystemTime=1731394078845 2024-11-12T06:47:58,875 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:47:58,875 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:47:58,876 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=0a0c05661b31342e3af97a578b0b00fc, regionState=OPEN, openSeqNum=2, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:47:58,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 because future has completed 2024-11-12T06:47:58,881 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-12T06:47:58,881 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 in 186 msec 2024-11-12T06:47:58,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-12T06:47:58,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0a0c05661b31342e3af97a578b0b00fc, ASSIGN in 343 msec 2024-11-12T06:47:58,883 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T06:47:58,883 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394078883"}]},"ts":"1731394078883"} 2024-11-12T06:47:58,885 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T06:47:58,886 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T06:47:58,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1840 sec 2024-11-12T06:47:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-12T06:47:59,849 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:47:59,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:59,852 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:47:59,852 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:47:59,855 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:47:59,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:47:59,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:47:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5af5b370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:47:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:47:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:47:59,858 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:47:59,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e548935, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:47:59,859 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:47:59,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:47:59,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:47:59,862 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:47:59,863 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T06:47:59,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 9f0beee3d6b6,43031,1731394046864 2024-11-12T06:47:59,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@67a4aeb8 2024-11-12T06:47:59,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T06:47:59,866 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T06:47:59,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T06:47:59,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T06:47:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-12T06:47:59,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741926_1102 (size=1097) 2024-11-12T06:48:00,297 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-12T06:48:00,297 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-12T06:48:00,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T06:48:00,311 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0a0c05661b31342e3af97a578b0b00fc, REOPEN/MOVE}] 2024-11-12T06:48:00,312 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0a0c05661b31342e3af97a578b0b00fc, REOPEN/MOVE 2024-11-12T06:48:00,313 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=0a0c05661b31342e3af97a578b0b00fc, regionState=CLOSING, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:00,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0a0c05661b31342e3af97a578b0b00fc, REOPEN/MOVE because future has completed 2024-11-12T06:48:00,315 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T06:48:00,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622}] 2024-11-12T06:48:00,469 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:00,470 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T06:48:00,470 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 0a0c05661b31342e3af97a578b0b00fc, disabling compactions & flushes 2024-11-12T06:48:00,470 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:00,470 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:00,470 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. after waiting 0 ms 2024-11-12T06:48:00,471 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:00,482 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T06:48:00,483 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:00,483 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 0a0c05661b31342e3af97a578b0b00fc: Waiting for close lock at 1731394080470Running coprocessor pre-close hooks at 1731394080470Disabling compacts and flushes for region at 1731394080470Disabling writes for close at 1731394080471 (+1 ms)Writing region close event to WAL at 1731394080472 (+1 ms)Running coprocessor post-close hooks at 1731394080483 (+11 ms)Closed at 1731394080483 2024-11-12T06:48:00,483 WARN [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: 0a0c05661b31342e3af97a578b0b00fc to self. 2024-11-12T06:48:00,485 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:00,485 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=0a0c05661b31342e3af97a578b0b00fc, regionState=CLOSED 2024-11-12T06:48:00,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 because future has completed 2024-11-12T06:48:00,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-12T06:48:00,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 in 173 msec 2024-11-12T06:48:00,490 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0a0c05661b31342e3af97a578b0b00fc, REOPEN/MOVE; state=CLOSED, location=9f0beee3d6b6,38777,1731394047622; forceNewPlan=false, retain=true 2024-11-12T06:48:00,641 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=0a0c05661b31342e3af97a578b0b00fc, regionState=OPENING, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:00,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0a0c05661b31342e3af97a578b0b00fc, REOPEN/MOVE because future has completed 2024-11-12T06:48:00,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622}] 2024-11-12T06:48:00,809 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:00,809 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} 2024-11-12T06:48:00,810 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:00,810 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T06:48:00,810 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:00,810 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:00,814 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:00,815 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:48:00,819 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a0c05661b31342e3af97a578b0b00fc columnFamilyName A 2024-11-12T06:48:00,821 DEBUG [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:00,821 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] regionserver.HStore(327): Store=0a0c05661b31342e3af97a578b0b00fc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:48:00,822 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:00,822 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:48:00,822 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a0c05661b31342e3af97a578b0b00fc columnFamilyName B 2024-11-12T06:48:00,823 DEBUG [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:00,823 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] regionserver.HStore(327): Store=0a0c05661b31342e3af97a578b0b00fc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:48:00,823 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:00,824 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:48:00,824 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a0c05661b31342e3af97a578b0b00fc columnFamilyName C 2024-11-12T06:48:00,824 DEBUG [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:00,825 INFO [StoreOpener-0a0c05661b31342e3af97a578b0b00fc-1 {}] regionserver.HStore(327): Store=0a0c05661b31342e3af97a578b0b00fc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:48:00,825 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:00,825 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:00,826 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:00,827 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:00,828 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:00,828 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:00,828 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T06:48:00,829 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:00,830 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 0a0c05661b31342e3af97a578b0b00fc; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66536295, jitterRate=-0.008531942963600159}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T06:48:00,830 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:00,831 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 0a0c05661b31342e3af97a578b0b00fc: Running coprocessor pre-open hook at 1731394080811Writing region info on filesystem at 1731394080811Initializing all the Stores at 1731394080812 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394080812Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394080813 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394080813Cleaning up temporary data from old regions at 1731394080828 (+15 ms)Running coprocessor post-open hooks at 1731394080830 (+2 ms)Region opened successfully at 1731394080831 (+1 ms) 2024-11-12T06:48:00,832 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., pid=35, masterSystemTime=1731394080800 2024-11-12T06:48:00,833 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:00,833 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:00,834 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=0a0c05661b31342e3af97a578b0b00fc, regionState=OPEN, openSeqNum=5, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:00,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 because future has completed 2024-11-12T06:48:00,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-12T06:48:00,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 in 190 msec 2024-11-12T06:48:00,840 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-12T06:48:00,840 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0a0c05661b31342e3af97a578b0b00fc, REOPEN/MOVE in 527 msec 2024-11-12T06:48:00,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-12T06:48:00,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 537 msec 2024-11-12T06:48:00,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 969 msec 2024-11-12T06:48:00,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-12T06:48:00,848 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:00,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,849 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:00,849 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:00,850 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:00,850 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:00,850 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:00,850 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60b0a416, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,850 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:00,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:00,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:00,852 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:00,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d9b2e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,853 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,853 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:00,853 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:00,854 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12988e5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:00,856 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:00,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38a83353, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,858 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,858 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:00,858 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:00,859 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cab7501, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:00,861 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8fe78f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,862 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:00,862 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:00,863 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70042b05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:00,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:00,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:00,865 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:00,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@622e3155, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,866 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:00,866 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:00,867 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24637948, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:00,868 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:00,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4895bd94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,870 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:00,870 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:00,871 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a2b3d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:00,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:00,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:00,873 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:00,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@332bc34b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,874 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:00,874 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:00,874 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761a8232, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:00,876 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:00,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@448192b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,877 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,877 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:00,877 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:00,878 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71441391, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:00,879 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4940a7d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:00,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:00,881 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c4b2ac9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:00,883 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:00,883 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3649d1c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:00,884 DEBUG [Thread-555 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:00,884 DEBUG [Thread-546 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:00,884 DEBUG [Thread-549 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:00,884 DEBUG [Thread-551 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:00,884 DEBUG [Thread-542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:00,884 DEBUG [Thread-538 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:00,885 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:48:00,885 DEBUG [Thread-540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:00,885 DEBUG [Thread-553 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:00,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:00,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:00,886 DEBUG [Thread-544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:00,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:00,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:00,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:00,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:00,886 DEBUG [Thread-540 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:00,886 DEBUG [Thread-540 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:00,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:00,887 DEBUG [Thread-544 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:00,887 DEBUG [Thread-544 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:00,887 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:00,887 DEBUG [Thread-553 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:00,887 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:00,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:00,888 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:00,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:00,888 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:00,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-12T06:48:00,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:00,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:00,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:00,889 DEBUG [Thread-553 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:00,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59706, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T06:48:00,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59718, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:00,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:00,890 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:48:00,891 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:48:00,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:48:00,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:48:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:48:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:48:00,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:48:00,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:48:00,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:48:00,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:48:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:48:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:48:00,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:00,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T06:48:00,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:00,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:00,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:00,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:00,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:00,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:00,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125d4c1841eb524d5d8616eac19efebc75_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394080900/Put/seqid=0 2024-11-12T06:48:00,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731394080944, max timestamp=1731394080958Num uniques 859; 2024-11-12T06:48:00,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731394080944, max timestamp=1731394080958Num uniques 859; 2024-11-12T06:48:00,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731394080944, max timestamp=1731394080958Num uniques 859; 2024-11-12T06:48:00,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T06:48:00,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T06:48:00,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T06:48:00,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741927_1103 (size=12154) 2024-11-12T06:48:00,981 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:00,988 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125d4c1841eb524d5d8616eac19efebc75_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125d4c1841eb524d5d8616eac19efebc75_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:00,991 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/09a884bf11824f28839444c806c7d788, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T06:48:01,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/09a884bf11824f28839444c806c7d788 is 175, key is test_row_0/A:col10/1731394080900/Put/seqid=0 2024-11-12T06:48:01,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394140996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394140998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394141000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741928_1104 (size=30955) 2024-11-12T06:48:01,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394141002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,017 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/09a884bf11824f28839444c806c7d788 2024-11-12T06:48:01,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394141006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T06:48:01,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:01,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:01,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:01,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:01,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:01,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/0262c680599c4c7db0f6cdafe6027a97 is 50, key is test_row_0/B:col10/1731394080900/Put/seqid=0 2024-11-12T06:48:01,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741929_1105 (size=12001) 2024-11-12T06:48:01,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/0262c680599c4c7db0f6cdafe6027a97 2024-11-12T06:48:01,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/8b41d993bea54a95af3c6bece42ca73a is 50, key is test_row_0/C:col10/1731394080900/Put/seqid=0 2024-11-12T06:48:01,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741930_1106 (size=12001) 2024-11-12T06:48:01,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394141117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394141118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394141118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394141127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394141137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T06:48:01,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:01,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:01,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:01,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:01,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:01,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T06:48:01,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394141337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394141337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394141337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394141337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394141347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T06:48:01,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:01,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:01,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:01,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:01,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:01,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/8b41d993bea54a95af3c6bece42ca73a 2024-11-12T06:48:01,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T06:48:01,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:01,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:01,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:01,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:01,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:01,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:01,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/09a884bf11824f28839444c806c7d788 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/09a884bf11824f28839444c806c7d788 2024-11-12T06:48:01,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/09a884bf11824f28839444c806c7d788, entries=150, sequenceid=14, filesize=30.2 K 2024-11-12T06:48:01,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/0262c680599c4c7db0f6cdafe6027a97 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/0262c680599c4c7db0f6cdafe6027a97 2024-11-12T06:48:01,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T06:48:01,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/0262c680599c4c7db0f6cdafe6027a97, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T06:48:01,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/8b41d993bea54a95af3c6bece42ca73a as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/8b41d993bea54a95af3c6bece42ca73a 2024-11-12T06:48:01,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/8b41d993bea54a95af3c6bece42ca73a, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T06:48:01,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0a0c05661b31342e3af97a578b0b00fc in 635ms, sequenceid=14, compaction requested=false 2024-11-12T06:48:01,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:01,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-12T06:48:01,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:01,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:01,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:01,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:01,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:01,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:01,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T06:48:01,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:01,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:01,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:01,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:01,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:01,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e86a7b99f5a04991b7535521cc42ce0a_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394080999/Put/seqid=0 2024-11-12T06:48:01,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394141675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394141675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741931_1107 (size=14594) 2024-11-12T06:48:01,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394141675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,688 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:01,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394141677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394141677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,697 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e86a7b99f5a04991b7535521cc42ce0a_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e86a7b99f5a04991b7535521cc42ce0a_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:01,699 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/2d4a0d21c5214e69a0bef834a15e8cc4, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:01,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/2d4a0d21c5214e69a0bef834a15e8cc4 is 175, key is test_row_0/A:col10/1731394080999/Put/seqid=0 2024-11-12T06:48:01,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741932_1108 (size=39549) 2024-11-12T06:48:01,712 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/2d4a0d21c5214e69a0bef834a15e8cc4 2024-11-12T06:48:01,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/1301bdc5312b4067b01236460bf5e338 is 50, key is test_row_0/B:col10/1731394080999/Put/seqid=0 2024-11-12T06:48:01,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741933_1109 (size=12001) 2024-11-12T06:48:01,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/1301bdc5312b4067b01236460bf5e338 2024-11-12T06:48:01,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/81b435dd7b8649c7a58c8ca038b4acdc is 50, key is test_row_0/C:col10/1731394080999/Put/seqid=0 2024-11-12T06:48:01,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741934_1110 (size=12001) 2024-11-12T06:48:01,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/81b435dd7b8649c7a58c8ca038b4acdc 2024-11-12T06:48:01,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394141788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394141788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/2d4a0d21c5214e69a0bef834a15e8cc4 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/2d4a0d21c5214e69a0bef834a15e8cc4 2024-11-12T06:48:01,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394141797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394141797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394141798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:01,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:01,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/2d4a0d21c5214e69a0bef834a15e8cc4, entries=200, sequenceid=50, filesize=38.6 K 2024-11-12T06:48:01,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/1301bdc5312b4067b01236460bf5e338 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/1301bdc5312b4067b01236460bf5e338 2024-11-12T06:48:01,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/1301bdc5312b4067b01236460bf5e338, entries=150, sequenceid=50, filesize=11.7 K 2024-11-12T06:48:01,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/81b435dd7b8649c7a58c8ca038b4acdc as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/81b435dd7b8649c7a58c8ca038b4acdc 2024-11-12T06:48:01,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T06:48:01,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:01,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:01,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:01,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:01,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:01,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/81b435dd7b8649c7a58c8ca038b4acdc, entries=150, sequenceid=50, filesize=11.7 K 2024-11-12T06:48:01,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=46.96 KB/48090 for 0a0c05661b31342e3af97a578b0b00fc in 168ms, sequenceid=50, compaction requested=false 2024-11-12T06:48:01,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:01,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T06:48:01,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:01,970 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T06:48:01,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:01,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:01,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:01,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:01,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:01,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:01,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c085453e8d6841a3b3409d6703a1f3d9_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394081674/Put/seqid=0 2024-11-12T06:48:01,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741935_1111 (size=12154) 2024-11-12T06:48:01,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:02,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:02,005 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c085453e8d6841a3b3409d6703a1f3d9_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c085453e8d6841a3b3409d6703a1f3d9_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:02,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/33640ee0daa348448e7bed1655a0622b, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:02,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/33640ee0daa348448e7bed1655a0622b is 175, key is test_row_0/A:col10/1731394081674/Put/seqid=0 2024-11-12T06:48:02,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741936_1112 (size=30955) 2024-11-12T06:48:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T06:48:02,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731394082019, max timestamp=1731394082034Num uniques 859; 2024-11-12T06:48:02,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T06:48:02,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731394082019, max timestamp=1731394082034Num uniques 859; 2024-11-12T06:48:02,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T06:48:02,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731394082019, max timestamp=1731394082034Num uniques 859; 2024-11-12T06:48:02,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T06:48:02,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394142066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:02,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394142067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:02,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:02,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:02,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394142069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:02,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:02,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:02,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394142070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:02,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:02,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:02,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394142070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:02,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:02,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:02,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394142187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:02,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:02,194 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394142187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:02,194 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,194 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:02,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:02,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394142187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:02,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:02,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:02,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394142188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:02,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:02,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:02,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:02,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394142188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:02,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:02,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394142397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:02,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:02,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394142397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:02,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:02,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:02,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394142397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:02,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:02,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394142407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:02,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:02,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394142407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:02,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:02,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:02,427 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/33640ee0daa348448e7bed1655a0622b 2024-11-12T06:48:02,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/3a73628fd52340e19727a626e2d439f3 is 50, key is test_row_0/B:col10/1731394081674/Put/seqid=0 2024-11-12T06:48:02,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741937_1113 (size=12001) 2024-11-12T06:48:02,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394142707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:02,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394142707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:02,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394142707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:02,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:02,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394142717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:02,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:02,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:02,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394142717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:02,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:02,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:02,841 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/3a73628fd52340e19727a626e2d439f3 2024-11-12T06:48:02,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/62d09e23835d466f85beea0af82093d1 is 50, key is test_row_0/C:col10/1731394081674/Put/seqid=0 2024-11-12T06:48:02,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741938_1114 (size=12001) 2024-11-12T06:48:02,860 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/62d09e23835d466f85beea0af82093d1 2024-11-12T06:48:02,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/33640ee0daa348448e7bed1655a0622b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/33640ee0daa348448e7bed1655a0622b 2024-11-12T06:48:02,872 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/33640ee0daa348448e7bed1655a0622b, entries=150, sequenceid=60, filesize=30.2 K 2024-11-12T06:48:02,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/3a73628fd52340e19727a626e2d439f3 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/3a73628fd52340e19727a626e2d439f3 2024-11-12T06:48:02,879 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/3a73628fd52340e19727a626e2d439f3, entries=150, sequenceid=60, filesize=11.7 K 2024-11-12T06:48:02,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/62d09e23835d466f85beea0af82093d1 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/62d09e23835d466f85beea0af82093d1 2024-11-12T06:48:02,886 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/62d09e23835d466f85beea0af82093d1, entries=150, sequenceid=60, filesize=11.7 K 2024-11-12T06:48:02,887 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0a0c05661b31342e3af97a578b0b00fc in 918ms, sequenceid=60, compaction requested=true 2024-11-12T06:48:02,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:02,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:02,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-12T06:48:02,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-12T06:48:02,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-12T06:48:02,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9980 sec 2024-11-12T06:48:02,893 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.0070 sec 2024-11-12T06:48:03,012 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T06:48:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T06:48:03,038 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:03,039 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:48:03,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-12T06:48:03,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T06:48:03,084 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:48:03,085 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:48:03,085 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:48:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T06:48:03,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-12T06:48:03,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:03,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:03,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:03,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:03,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:03,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:03,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111264b88e42c351455eb294b35d72378bd7_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394082064/Put/seqid=0 2024-11-12T06:48:03,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T06:48:03,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:03,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:03,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:03,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:03,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:03,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:03,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394143238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394143240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:03,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394143242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:03,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394143249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:03,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394143250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:03,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:03,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:03,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:03,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741939_1115 (size=14594) 2024-11-12T06:48:03,266 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:03,276 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111264b88e42c351455eb294b35d72378bd7_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111264b88e42c351455eb294b35d72378bd7_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:03,278 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/0e5180caf5134c40b85725d293691dd0, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:03,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/0e5180caf5134c40b85725d293691dd0 is 175, key is test_row_0/A:col10/1731394082064/Put/seqid=0 2024-11-12T06:48:03,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741940_1116 (size=39549) 2024-11-12T06:48:03,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394143357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:03,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:03,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394143357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:03,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:03,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394143357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:03,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:03,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394143358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:03,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:03,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394143358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:03,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:03,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T06:48:03,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T06:48:03,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:03,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:03,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:03,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:03,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:03,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T06:48:03,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:03,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:03,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:03,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:03,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:03,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:03,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394143567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:03,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394143567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:03,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:03,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394143567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:03,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394143567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:03,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:03,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394143577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:03,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:03,693 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/0e5180caf5134c40b85725d293691dd0 2024-11-12T06:48:03,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/5af07f9f0ccc43109f970425c5aa0d6f is 50, key is test_row_0/B:col10/1731394082064/Put/seqid=0 2024-11-12T06:48:03,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741941_1117 (size=12001) 2024-11-12T06:48:03,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T06:48:03,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:03,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:03,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:03,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:03,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T06:48:03,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:03,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T06:48:03,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:03,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:03,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:03,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:03,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:03,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394143877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:03,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394143877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:03,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394143877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394143877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:03,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:03,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394143887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:03,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:03,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:04,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T06:48:04,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:04,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:04,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:04,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:04,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:04,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:04,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/5af07f9f0ccc43109f970425c5aa0d6f 2024-11-12T06:48:04,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/0beb0b1fa7dd4ab19cd461bf1135d19f is 50, key is test_row_0/C:col10/1731394082064/Put/seqid=0 2024-11-12T06:48:04,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741942_1118 (size=12001) 2024-11-12T06:48:04,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/0beb0b1fa7dd4ab19cd461bf1135d19f 2024-11-12T06:48:04,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/0e5180caf5134c40b85725d293691dd0 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/0e5180caf5134c40b85725d293691dd0 2024-11-12T06:48:04,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/0e5180caf5134c40b85725d293691dd0, entries=200, sequenceid=96, filesize=38.6 K 2024-11-12T06:48:04,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/5af07f9f0ccc43109f970425c5aa0d6f as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/5af07f9f0ccc43109f970425c5aa0d6f 2024-11-12T06:48:04,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/5af07f9f0ccc43109f970425c5aa0d6f, entries=150, sequenceid=96, filesize=11.7 K 2024-11-12T06:48:04,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/0beb0b1fa7dd4ab19cd461bf1135d19f as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/0beb0b1fa7dd4ab19cd461bf1135d19f 2024-11-12T06:48:04,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/0beb0b1fa7dd4ab19cd461bf1135d19f, entries=150, sequenceid=96, filesize=11.7 K 2024-11-12T06:48:04,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=46.96 KB/48090 for 0a0c05661b31342e3af97a578b0b00fc in 915ms, sequenceid=96, compaction requested=true 2024-11-12T06:48:04,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:04,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:48:04,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:04,142 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:48:04,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:48:04,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:04,142 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:48:04,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:48:04,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:04,144 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:48:04,144 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/B is initiating minor compaction (all files) 2024-11-12T06:48:04,144 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/B in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:04,144 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/0262c680599c4c7db0f6cdafe6027a97, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/1301bdc5312b4067b01236460bf5e338, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/3a73628fd52340e19727a626e2d439f3, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/5af07f9f0ccc43109f970425c5aa0d6f] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=46.9 K 2024-11-12T06:48:04,144 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 0262c680599c4c7db0f6cdafe6027a97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731394080895 2024-11-12T06:48:04,145 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 1301bdc5312b4067b01236460bf5e338, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731394080999 2024-11-12T06:48:04,145 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a73628fd52340e19727a626e2d439f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731394081661 2024-11-12T06:48:04,145 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 5af07f9f0ccc43109f970425c5aa0d6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731394082064 2024-11-12T06:48:04,145 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141008 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:48:04,145 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/A is initiating minor compaction (all files) 2024-11-12T06:48:04,146 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/A in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:04,146 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/09a884bf11824f28839444c806c7d788, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/2d4a0d21c5214e69a0bef834a15e8cc4, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/33640ee0daa348448e7bed1655a0622b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/0e5180caf5134c40b85725d293691dd0] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=137.7 K 2024-11-12T06:48:04,146 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:04,146 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. files: [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/09a884bf11824f28839444c806c7d788, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/2d4a0d21c5214e69a0bef834a15e8cc4, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/33640ee0daa348448e7bed1655a0622b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/0e5180caf5134c40b85725d293691dd0] 2024-11-12T06:48:04,147 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09a884bf11824f28839444c806c7d788, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731394080895 2024-11-12T06:48:04,147 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d4a0d21c5214e69a0bef834a15e8cc4, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731394080987 2024-11-12T06:48:04,148 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33640ee0daa348448e7bed1655a0622b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731394081661 2024-11-12T06:48:04,148 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e5180caf5134c40b85725d293691dd0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731394082064 2024-11-12T06:48:04,159 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#B#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:04,160 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/ecbb9fa8b2da41a69f33cccab7ecb57d is 50, key is test_row_0/B:col10/1731394082064/Put/seqid=0 2024-11-12T06:48:04,165 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:04,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T06:48:04,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:04,167 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T06:48:04,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:04,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:04,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:04,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:04,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:04,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:04,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741943_1119 (size=12139) 2024-11-12T06:48:04,170 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112aa268f3c315f49f582d2aaca3993db78_0a0c05661b31342e3af97a578b0b00fc store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:04,176 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112aa268f3c315f49f582d2aaca3993db78_0a0c05661b31342e3af97a578b0b00fc, store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:04,177 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112aa268f3c315f49f582d2aaca3993db78_0a0c05661b31342e3af97a578b0b00fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:04,181 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/ecbb9fa8b2da41a69f33cccab7ecb57d as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/ecbb9fa8b2da41a69f33cccab7ecb57d 2024-11-12T06:48:04,193 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/B of 0a0c05661b31342e3af97a578b0b00fc into ecbb9fa8b2da41a69f33cccab7ecb57d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:04,193 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:04,193 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/B, priority=12, startTime=1731394084142; duration=0sec 2024-11-12T06:48:04,193 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:04,193 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:B 2024-11-12T06:48:04,194 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:48:04,195 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:48:04,195 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/C is initiating minor compaction (all files) 2024-11-12T06:48:04,195 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/C in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:04,196 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/8b41d993bea54a95af3c6bece42ca73a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/81b435dd7b8649c7a58c8ca038b4acdc, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/62d09e23835d466f85beea0af82093d1, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/0beb0b1fa7dd4ab19cd461bf1135d19f] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=46.9 K 2024-11-12T06:48:04,196 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b41d993bea54a95af3c6bece42ca73a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731394080895 2024-11-12T06:48:04,196 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 81b435dd7b8649c7a58c8ca038b4acdc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731394080999 2024-11-12T06:48:04,197 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 62d09e23835d466f85beea0af82093d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731394081661 2024-11-12T06:48:04,197 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 0beb0b1fa7dd4ab19cd461bf1135d19f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731394082064 2024-11-12T06:48:04,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e481007355ca4e7d831c829c63ef3de2_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394083235/Put/seqid=0 2024-11-12T06:48:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T06:48:04,229 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#C#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:04,229 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/f766cd8c416c4806ba23243d47fa4f07 is 50, key is test_row_0/C:col10/1731394082064/Put/seqid=0 2024-11-12T06:48:04,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741944_1120 (size=4469) 2024-11-12T06:48:04,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741945_1121 (size=12154) 2024-11-12T06:48:04,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:04,248 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e481007355ca4e7d831c829c63ef3de2_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e481007355ca4e7d831c829c63ef3de2_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:04,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/8632999e62a14116a5e59a841c616e59, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:04,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/8632999e62a14116a5e59a841c616e59 is 175, key is test_row_0/A:col10/1731394083235/Put/seqid=0 2024-11-12T06:48:04,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741946_1122 (size=12139) 2024-11-12T06:48:04,264 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/f766cd8c416c4806ba23243d47fa4f07 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/f766cd8c416c4806ba23243d47fa4f07 2024-11-12T06:48:04,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741947_1123 (size=30955) 2024-11-12T06:48:04,268 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=106, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/8632999e62a14116a5e59a841c616e59 2024-11-12T06:48:04,274 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/C of 0a0c05661b31342e3af97a578b0b00fc into f766cd8c416c4806ba23243d47fa4f07(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:04,275 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:04,275 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/C, priority=12, startTime=1731394084142; duration=0sec 2024-11-12T06:48:04,276 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:04,276 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:C 2024-11-12T06:48:04,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/6173a94b3ac642b8b6b03e49326b2215 is 50, key is test_row_0/B:col10/1731394083235/Put/seqid=0 2024-11-12T06:48:04,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741948_1124 (size=12001) 2024-11-12T06:48:04,305 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/6173a94b3ac642b8b6b03e49326b2215 2024-11-12T06:48:04,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/141b19a23fe3429a99d04ea9bff2c0c5 is 50, key is test_row_0/C:col10/1731394083235/Put/seqid=0 2024-11-12T06:48:04,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741949_1125 (size=12001) 2024-11-12T06:48:04,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:04,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:04,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731394084395, max timestamp=1731394084411Num uniques 859; 2024-11-12T06:48:04,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731394084395, max timestamp=1731394084411Num uniques 859; 2024-11-12T06:48:04,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-12T06:48:04,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-12T06:48:04,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394084405, max timestamp=1731394084411Num uniques 859; 2024-11-12T06:48:04,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:48:04,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394144429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:04,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:04,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394144430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:04,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:04,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:04,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394144431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:04,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:04,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:04,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:04,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394144437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:04,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:04,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:04,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394144440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:04,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:04,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394144547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:04,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394144547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394144548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:04,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394144548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:04,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:04,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394144557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:04,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:04,635 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#A#compaction#100 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:04,636 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/c51192ee6be44deb825dc498f80fcc91 is 175, key is test_row_0/A:col10/1731394082064/Put/seqid=0 2024-11-12T06:48:04,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741950_1126 (size=31093) 2024-11-12T06:48:04,653 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/c51192ee6be44deb825dc498f80fcc91 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/c51192ee6be44deb825dc498f80fcc91 2024-11-12T06:48:04,661 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/A of 0a0c05661b31342e3af97a578b0b00fc into c51192ee6be44deb825dc498f80fcc91(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:04,661 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:04,661 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/A, priority=12, startTime=1731394084142; duration=0sec 2024-11-12T06:48:04,661 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:04,662 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:A 2024-11-12T06:48:04,737 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/141b19a23fe3429a99d04ea9bff2c0c5 2024-11-12T06:48:04,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/8632999e62a14116a5e59a841c616e59 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/8632999e62a14116a5e59a841c616e59 2024-11-12T06:48:04,748 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/8632999e62a14116a5e59a841c616e59, entries=150, sequenceid=106, filesize=30.2 K 2024-11-12T06:48:04,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/6173a94b3ac642b8b6b03e49326b2215 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/6173a94b3ac642b8b6b03e49326b2215 2024-11-12T06:48:04,755 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/6173a94b3ac642b8b6b03e49326b2215, entries=150, sequenceid=106, filesize=11.7 K 2024-11-12T06:48:04,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/141b19a23fe3429a99d04ea9bff2c0c5 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/141b19a23fe3429a99d04ea9bff2c0c5 2024-11-12T06:48:04,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:04,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394144757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:04,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:04,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394144757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:04,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:04,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394144757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:04,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:04,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:04,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:04,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:04,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394144757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:04,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:04,764 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/141b19a23fe3429a99d04ea9bff2c0c5, entries=150, sequenceid=106, filesize=11.7 K 2024-11-12T06:48:04,765 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 0a0c05661b31342e3af97a578b0b00fc in 598ms, sequenceid=106, compaction requested=false 2024-11-12T06:48:04,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:04,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:04,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-12T06:48:04,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-12T06:48:04,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-12T06:48:04,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6820 sec 2024-11-12T06:48:04,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:04,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=154.31 KB heapSize=426 KB 2024-11-12T06:48:04,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:04,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:04,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:04,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:04,772 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.7320 sec 2024-11-12T06:48:04,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:04,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:04,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111238a140d5f32c41cb9c1ced4b3feb9737_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394084769/Put/seqid=0 2024-11-12T06:48:04,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741951_1127 (size=14794) 2024-11-12T06:48:04,794 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:04,806 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111238a140d5f32c41cb9c1ced4b3feb9737_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111238a140d5f32c41cb9c1ced4b3feb9737_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:04,807 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/4b392ea52e464c34adf31bf93b97e663, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:04,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/4b392ea52e464c34adf31bf93b97e663 is 175, key is test_row_0/A:col10/1731394084769/Put/seqid=0 2024-11-12T06:48:04,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741952_1128 (size=39749) 2024-11-12T06:48:04,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:04,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394144809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:04,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:04,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:04,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394144927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:04,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:04,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:05,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394145067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:05,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:05,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394145067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:05,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:05,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:05,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394145067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:05,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:05,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:05,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:05,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:05,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394145067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:05,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:05,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:05,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:05,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:05,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:05,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394145137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:05,218 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/4b392ea52e464c34adf31bf93b97e663 2024-11-12T06:48:05,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/a5fa4322e4634658a7bb29b09e6279e5 is 50, key is test_row_0/B:col10/1731394084769/Put/seqid=0 2024-11-12T06:48:05,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T06:48:05,227 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:05,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:48:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-12T06:48:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T06:48:05,229 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:48:05,230 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:48:05,230 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:48:05,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741953_1129 (size=12151) 2024-11-12T06:48:05,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/a5fa4322e4634658a7bb29b09e6279e5 2024-11-12T06:48:05,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/5717f2cc67544f66b04547215f1ff25b is 50, key is test_row_0/C:col10/1731394084769/Put/seqid=0 2024-11-12T06:48:05,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741954_1130 (size=12151) 2024-11-12T06:48:05,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/5717f2cc67544f66b04547215f1ff25b 2024-11-12T06:48:05,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/4b392ea52e464c34adf31bf93b97e663 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/4b392ea52e464c34adf31bf93b97e663 2024-11-12T06:48:05,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/4b392ea52e464c34adf31bf93b97e663, entries=200, sequenceid=145, filesize=38.8 K 2024-11-12T06:48:05,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/a5fa4322e4634658a7bb29b09e6279e5 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/a5fa4322e4634658a7bb29b09e6279e5 2024-11-12T06:48:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/a5fa4322e4634658a7bb29b09e6279e5, entries=150, sequenceid=145, filesize=11.9 K 2024-11-12T06:48:05,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/5717f2cc67544f66b04547215f1ff25b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/5717f2cc67544f66b04547215f1ff25b 2024-11-12T06:48:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/5717f2cc67544f66b04547215f1ff25b, entries=150, sequenceid=145, filesize=11.9 K 2024-11-12T06:48:05,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~425.95 KB/436176, currentSize=33.54 KB/34350 for 0a0c05661b31342e3af97a578b0b00fc in 532ms, sequenceid=145, compaction requested=true 2024-11-12T06:48:05,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:05,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:48:05,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:05,305 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:05,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:48:05,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:05,305 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:05,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:48:05,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:05,306 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:05,306 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:05,306 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/B is initiating minor compaction (all files) 2024-11-12T06:48:05,306 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/A is initiating minor compaction (all files) 2024-11-12T06:48:05,306 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/B in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:05,306 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/A in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:05,306 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/c51192ee6be44deb825dc498f80fcc91, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/8632999e62a14116a5e59a841c616e59, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/4b392ea52e464c34adf31bf93b97e663] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=99.4 K 2024-11-12T06:48:05,306 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/ecbb9fa8b2da41a69f33cccab7ecb57d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/6173a94b3ac642b8b6b03e49326b2215, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/a5fa4322e4634658a7bb29b09e6279e5] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=35.4 K 2024-11-12T06:48:05,306 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:05,306 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. files: [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/c51192ee6be44deb825dc498f80fcc91, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/8632999e62a14116a5e59a841c616e59, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/4b392ea52e464c34adf31bf93b97e663] 2024-11-12T06:48:05,307 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting ecbb9fa8b2da41a69f33cccab7ecb57d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731394082064 2024-11-12T06:48:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,310 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting c51192ee6be44deb825dc498f80fcc91, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731394082064 2024-11-12T06:48:05,310 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 6173a94b3ac642b8b6b03e49326b2215, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731394083235 2024-11-12T06:48:05,311 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8632999e62a14116a5e59a841c616e59, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731394083235 2024-11-12T06:48:05,311 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting a5fa4322e4634658a7bb29b09e6279e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731394084429 2024-11-12T06:48:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,312 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b392ea52e464c34adf31bf93b97e663, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731394084429 2024-11-12T06:48:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,325 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,337 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#B#compaction#109 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T06:48:05,337 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/7eac3c49bc9b479a9adc5c025d666fa0 is 50, key is test_row_0/B:col10/1731394084769/Put/seqid=0 2024-11-12T06:48:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,339 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112dc3886bc41ef4a63af3beee336de4325_0a0c05661b31342e3af97a578b0b00fc store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,344 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112dc3886bc41ef4a63af3beee336de4325_0a0c05661b31342e3af97a578b0b00fc, store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:05,344 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112dc3886bc41ef4a63af3beee336de4325_0a0c05661b31342e3af97a578b0b00fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741955_1131 (size=12391) 2024-11-12T06:48:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741956_1132 (size=4469) 2024-11-12T06:48:05,378 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#A#compaction#108 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,379 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/8169947025c84b9eb46bba283c4cef11 is 175, key is test_row_0/A:col10/1731394084769/Put/seqid=0 2024-11-12T06:48:05,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T06:48:05,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:05,384 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-12T06:48:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:05,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:05,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:05,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:05,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:05,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741957_1133 (size=31345) 2024-11-12T06:48:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,427 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/8169947025c84b9eb46bba283c4cef11 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/8169947025c84b9eb46bba283c4cef11 2024-11-12T06:48:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,435 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/A of 0a0c05661b31342e3af97a578b0b00fc into 8169947025c84b9eb46bba283c4cef11(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:05,435 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:05,435 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/A, priority=13, startTime=1731394085305; duration=0sec 2024-11-12T06:48:05,435 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:05,435 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:A 2024-11-12T06:48:05,435 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,437 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,437 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/C is initiating minor compaction (all files) 2024-11-12T06:48:05,437 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/C in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:05,437 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/f766cd8c416c4806ba23243d47fa4f07, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/141b19a23fe3429a99d04ea9bff2c0c5, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/5717f2cc67544f66b04547215f1ff25b] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=35.4 K 2024-11-12T06:48:05,438 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting f766cd8c416c4806ba23243d47fa4f07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731394082064 2024-11-12T06:48:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,438 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 141b19a23fe3429a99d04ea9bff2c0c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731394083235 2024-11-12T06:48:05,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,440 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5717f2cc67544f66b04547215f1ff25b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731394084429 2024-11-12T06:48:05,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112181cb2533b1542949d9302bf523618ab_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394084798/Put/seqid=0 2024-11-12T06:48:05,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741958_1134 (size=9814) 2024-11-12T06:48:05,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:05,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:05,491 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#C#compaction#111 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:05,492 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/151d8c5d154741afbbaf8a2fa989cfb2 is 50, key is test_row_0/C:col10/1731394084769/Put/seqid=0 2024-11-12T06:48:05,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741959_1135 (size=12391) 2024-11-12T06:48:05,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T06:48:05,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731394085586, max timestamp=1731394085598Num uniques 859; 2024-11-12T06:48:05,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T06:48:05,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731394085586, max timestamp=1731394085598Num uniques 859; 2024-11-12T06:48:05,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-12T06:48:05,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731394085586, max timestamp=1731394085598Num uniques 859; 2024-11-12T06:48:05,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T06:48:05,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731394085607, max timestamp=1731394085644Num uniques 1118; 2024-11-12T06:48:05,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731394085607, max timestamp=1731394085644Num uniques 1118; 2024-11-12T06:48:05,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-11-12T06:48:05,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-11-12T06:48:05,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731394085607, max timestamp=1731394085644Num uniques 1009; 2024-11-12T06:48:05,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=14558, suffix heap size=132904,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=318 2024-11-12T06:48:05,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731394085686, max timestamp=1731394085691Num uniques 1177; 2024-11-12T06:48:05,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=8102, suffix heap size=150376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=177 2024-11-12T06:48:05,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731394085686, max timestamp=1731394085691Num uniques 1177; 2024-11-12T06:48:05,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=8102, suffix heap size=150376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=177 2024-11-12T06:48:05,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731394085686, max timestamp=1731394085691Num uniques 1177; 2024-11-12T06:48:05,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=8102, suffix heap size=150376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=177 2024-11-12T06:48:05,788 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/7eac3c49bc9b479a9adc5c025d666fa0 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/7eac3c49bc9b479a9adc5c025d666fa0 2024-11-12T06:48:05,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1731394085723, max timestamp=1731394085805Num uniques 1036; 2024-11-12T06:48:05,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1731394085723, max timestamp=1731394085805Num uniques 1036; 2024-11-12T06:48:05,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=17676, suffix heap size=135712,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=386 2024-11-12T06:48:05,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=17676, suffix heap size=135712,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=386 2024-11-12T06:48:05,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1731394085723, max timestamp=1731394085805Num uniques 1036; 2024-11-12T06:48:05,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=17676, suffix heap size=135712,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=386 2024-11-12T06:48:05,822 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/B of 0a0c05661b31342e3af97a578b0b00fc into 7eac3c49bc9b479a9adc5c025d666fa0(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:05,822 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:05,823 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/B, priority=13, startTime=1731394085305; duration=0sec 2024-11-12T06:48:05,823 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:05,823 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:B 2024-11-12T06:48:05,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T06:48:05,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731394085835, max timestamp=1731394085857Num uniques 1245; 2024-11-12T06:48:05,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731394085835, max timestamp=1731394085857Num uniques 1245; 2024-11-12T06:48:05,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=57020, new segment data size=13510, suffix heap size=157448,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1245, new segment cells count=295 2024-11-12T06:48:05,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=57020, new segment data size=13510, suffix heap size=157448,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1245, new segment cells count=295 2024-11-12T06:48:05,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731394085852, max timestamp=1731394085857Num uniques 1245; 2024-11-12T06:48:05,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=57020, new segment data size=8930, suffix heap size=157448,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1245, new segment cells count=195 2024-11-12T06:48:05,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:05,898 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112181cb2533b1542949d9302bf523618ab_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112181cb2533b1542949d9302bf523618ab_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:05,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/68021602b6ba43aa8e215cfcbc74ae03, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:05,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/68021602b6ba43aa8e215cfcbc74ae03 is 175, key is test_row_0/A:col10/1731394084798/Put/seqid=0 2024-11-12T06:48:05,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741960_1136 (size=22461) 2024-11-12T06:48:05,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394145906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:05,912 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/68021602b6ba43aa8e215cfcbc74ae03 2024-11-12T06:48:05,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394145908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:05,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:05,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:05,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:05,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:05,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:05,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:05,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394145909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:05,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394145910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:05,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394145912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:05,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/a5e0381ef6f341e6aafd5b26b2c4ff15 is 50, key is test_row_0/B:col10/1731394084798/Put/seqid=0 2024-11-12T06:48:05,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741961_1137 (size=9757) 2024-11-12T06:48:05,945 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/a5e0381ef6f341e6aafd5b26b2c4ff15 2024-11-12T06:48:05,952 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/151d8c5d154741afbbaf8a2fa989cfb2 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/151d8c5d154741afbbaf8a2fa989cfb2 2024-11-12T06:48:05,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/43e9be02ca0b4d819e1522e02385bff1 is 50, key is test_row_0/C:col10/1731394084798/Put/seqid=0 2024-11-12T06:48:05,965 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/C of 0a0c05661b31342e3af97a578b0b00fc into 151d8c5d154741afbbaf8a2fa989cfb2(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:05,965 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:05,965 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/C, priority=13, startTime=1731394085305; duration=0sec 2024-11-12T06:48:05,965 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:05,965 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:C 2024-11-12T06:48:05,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741962_1138 (size=9757) 2024-11-12T06:48:05,979 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/43e9be02ca0b4d819e1522e02385bff1 2024-11-12T06:48:05,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/68021602b6ba43aa8e215cfcbc74ae03 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/68021602b6ba43aa8e215cfcbc74ae03 2024-11-12T06:48:05,993 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/68021602b6ba43aa8e215cfcbc74ae03, entries=100, sequenceid=153, filesize=21.9 K 2024-11-12T06:48:05,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/a5e0381ef6f341e6aafd5b26b2c4ff15 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/a5e0381ef6f341e6aafd5b26b2c4ff15 2024-11-12T06:48:06,001 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/a5e0381ef6f341e6aafd5b26b2c4ff15, entries=100, sequenceid=153, filesize=9.5 K 2024-11-12T06:48:06,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/43e9be02ca0b4d819e1522e02385bff1 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/43e9be02ca0b4d819e1522e02385bff1 2024-11-12T06:48:06,018 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/43e9be02ca0b4d819e1522e02385bff1, entries=100, sequenceid=153, filesize=9.5 K 2024-11-12T06:48:06,020 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=149.83 KB/153430 for 0a0c05661b31342e3af97a578b0b00fc in 636ms, sequenceid=153, compaction requested=false 2024-11-12T06:48:06,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:06,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:06,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-12T06:48:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-12T06:48:06,025 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-12T06:48:06,025 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 792 msec 2024-11-12T06:48:06,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731394085887, max timestamp=1731394086023Num uniques 1154; 2024-11-12T06:48:06,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 799 msec 2024-11-12T06:48:06,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-11-12T06:48:06,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=111.82 KB heapSize=313.83 KB 2024-11-12T06:48:06,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731394085887, max timestamp=1731394086023Num uniques 1154; 2024-11-12T06:48:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:06,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-11-12T06:48:06,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:06,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:06,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:06,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:06,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:06,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:48:06,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129e812c611d8c452083ec998b65e4a6a7_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394085910/Put/seqid=0 2024-11-12T06:48:06,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394146079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394146079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394146079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394146081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394146081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741963_1139 (size=17484) 2024-11-12T06:48:06,105 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:06,113 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129e812c611d8c452083ec998b65e4a6a7_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129e812c611d8c452083ec998b65e4a6a7_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:06,115 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/3d76a865dd3a423db2252cdad92741c8, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:06,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/3d76a865dd3a423db2252cdad92741c8 is 175, key is test_row_0/A:col10/1731394085910/Put/seqid=0 2024-11-12T06:48:06,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741964_1140 (size=48589) 2024-11-12T06:48:06,129 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=264, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/3d76a865dd3a423db2252cdad92741c8 2024-11-12T06:48:06,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/63e7552c7cae4ccbbd344b3e846abf00 is 50, key is test_row_0/B:col10/1731394085910/Put/seqid=0 2024-11-12T06:48:06,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741965_1141 (size=12301) 2024-11-12T06:48:06,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/63e7552c7cae4ccbbd344b3e846abf00 2024-11-12T06:48:06,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/b37835d6e90c4ac2894f171e8ad09e5d is 50, key is test_row_0/C:col10/1731394085910/Put/seqid=0 2024-11-12T06:48:06,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741966_1142 (size=12301) 2024-11-12T06:48:06,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394146197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394146197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394146197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394146197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394146197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T06:48:06,367 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:06,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:48:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-12T06:48:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T06:48:06,369 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:48:06,370 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:48:06,370 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:48:06,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394146407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394146407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394146407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394146408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394146408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T06:48:06,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T06:48:06,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:06,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:06,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:06,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:06,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:06,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/b37835d6e90c4ac2894f171e8ad09e5d 2024-11-12T06:48:06,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/3d76a865dd3a423db2252cdad92741c8 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/3d76a865dd3a423db2252cdad92741c8 2024-11-12T06:48:06,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/3d76a865dd3a423db2252cdad92741c8, entries=250, sequenceid=264, filesize=47.5 K 2024-11-12T06:48:06,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/63e7552c7cae4ccbbd344b3e846abf00 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/63e7552c7cae4ccbbd344b3e846abf00 2024-11-12T06:48:06,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/63e7552c7cae4ccbbd344b3e846abf00, entries=150, sequenceid=264, filesize=12.0 K 2024-11-12T06:48:06,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/b37835d6e90c4ac2894f171e8ad09e5d as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/b37835d6e90c4ac2894f171e8ad09e5d 2024-11-12T06:48:06,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/b37835d6e90c4ac2894f171e8ad09e5d, entries=150, sequenceid=264, filesize=12.0 K 2024-11-12T06:48:06,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~205.06 KB/209984, currentSize=107.34 KB/109920 for 0a0c05661b31342e3af97a578b0b00fc in 576ms, sequenceid=264, compaction requested=true 2024-11-12T06:48:06,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:06,606 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:06,608 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102395 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:06,608 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/A is initiating minor compaction (all files) 2024-11-12T06:48:06,608 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/A in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:06,608 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/8169947025c84b9eb46bba283c4cef11, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/68021602b6ba43aa8e215cfcbc74ae03, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/3d76a865dd3a423db2252cdad92741c8] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=100.0 K 2024-11-12T06:48:06,608 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:06,608 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. files: [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/8169947025c84b9eb46bba283c4cef11, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/68021602b6ba43aa8e215cfcbc74ae03, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/3d76a865dd3a423db2252cdad92741c8] 2024-11-12T06:48:06,609 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8169947025c84b9eb46bba283c4cef11, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731394084429 2024-11-12T06:48:06,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:48:06,609 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68021602b6ba43aa8e215cfcbc74ae03, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731394084798 2024-11-12T06:48:06,609 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d76a865dd3a423db2252cdad92741c8, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731394085887 2024-11-12T06:48:06,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:06,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:48:06,611 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:06,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:06,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:48:06,615 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:06,615 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/B is initiating minor compaction (all files) 2024-11-12T06:48:06,615 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/B in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:06,615 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/7eac3c49bc9b479a9adc5c025d666fa0, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/a5e0381ef6f341e6aafd5b26b2c4ff15, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/63e7552c7cae4ccbbd344b3e846abf00] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=33.6 K 2024-11-12T06:48:06,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:06,616 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 7eac3c49bc9b479a9adc5c025d666fa0, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731394084429 2024-11-12T06:48:06,616 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting a5e0381ef6f341e6aafd5b26b2c4ff15, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731394084798 2024-11-12T06:48:06,617 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 63e7552c7cae4ccbbd344b3e846abf00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731394085910 2024-11-12T06:48:06,619 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:06,621 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112800cbcb97cc04d42b017a2b1040d6659_0a0c05661b31342e3af97a578b0b00fc store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:06,624 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112800cbcb97cc04d42b017a2b1040d6659_0a0c05661b31342e3af97a578b0b00fc, store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:06,624 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112800cbcb97cc04d42b017a2b1040d6659_0a0c05661b31342e3af97a578b0b00fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:06,628 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#B#compaction#118 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:06,628 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/0a6f5d8b908249f2bfc25fc56037b4a3 is 50, key is test_row_0/B:col10/1731394085910/Put/seqid=0 2024-11-12T06:48:06,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741967_1143 (size=4469) 2024-11-12T06:48:06,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741968_1144 (size=12643) 2024-11-12T06:48:06,650 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#A#compaction#117 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:06,651 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/860331e24d564b50a4ac26400bd2549d is 175, key is test_row_0/A:col10/1731394085910/Put/seqid=0 2024-11-12T06:48:06,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741969_1145 (size=31597) 2024-11-12T06:48:06,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T06:48:06,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:06,675 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T06:48:06,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:06,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:06,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:06,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:06,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:06,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:06,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124b5171c6bead4860aff48ed518cd8327_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394086078/Put/seqid=0 2024-11-12T06:48:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T06:48:06,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741970_1146 (size=12454) 2024-11-12T06:48:06,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:06,699 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124b5171c6bead4860aff48ed518cd8327_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124b5171c6bead4860aff48ed518cd8327_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:06,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/b1067816533b4346acef019a06dcf7e0, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:06,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/b1067816533b4346acef019a06dcf7e0 is 175, key is test_row_0/A:col10/1731394086078/Put/seqid=0 2024-11-12T06:48:06,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741971_1147 (size=31255) 2024-11-12T06:48:06,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:06,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:06,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394146726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394146727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394146728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394146729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394146729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394146837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394146837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394146837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394146837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:06,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394146847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T06:48:07,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394147047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:07,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394147047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:07,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394147047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394147047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:07,057 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/0a6f5d8b908249f2bfc25fc56037b4a3 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/0a6f5d8b908249f2bfc25fc56037b4a3 2024-11-12T06:48:07,067 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/B of 0a0c05661b31342e3af97a578b0b00fc into 0a6f5d8b908249f2bfc25fc56037b4a3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:07,067 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:07,067 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/B, priority=13, startTime=1731394086611; duration=0sec 2024-11-12T06:48:07,067 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:07,068 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:B 2024-11-12T06:48:07,068 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:07,071 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:07,071 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/C is initiating minor compaction (all files) 2024-11-12T06:48:07,071 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/C in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:07,071 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/151d8c5d154741afbbaf8a2fa989cfb2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/43e9be02ca0b4d819e1522e02385bff1, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/b37835d6e90c4ac2894f171e8ad09e5d] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=33.6 K 2024-11-12T06:48:07,072 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 151d8c5d154741afbbaf8a2fa989cfb2, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731394084429 2024-11-12T06:48:07,072 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/860331e24d564b50a4ac26400bd2549d as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/860331e24d564b50a4ac26400bd2549d 2024-11-12T06:48:07,072 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 43e9be02ca0b4d819e1522e02385bff1, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731394084798 2024-11-12T06:48:07,073 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting b37835d6e90c4ac2894f171e8ad09e5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731394085910 2024-11-12T06:48:07,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394147069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:07,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:07,084 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/A of 0a0c05661b31342e3af97a578b0b00fc into 860331e24d564b50a4ac26400bd2549d(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:07,084 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:07,084 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/A, priority=13, startTime=1731394086606; duration=0sec 2024-11-12T06:48:07,085 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:07,085 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:A 2024-11-12T06:48:07,086 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#C#compaction#120 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:07,087 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/35d6a7259d7e4efe907e7d281b1cbde8 is 50, key is test_row_0/C:col10/1731394085910/Put/seqid=0 2024-11-12T06:48:07,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741972_1148 (size=12643) 2024-11-12T06:48:07,115 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=283, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/b1067816533b4346acef019a06dcf7e0 2024-11-12T06:48:07,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/52166ede5574482a91586d16e0f14c87 is 50, key is test_row_0/B:col10/1731394086078/Put/seqid=0 2024-11-12T06:48:07,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741973_1149 (size=12301) 2024-11-12T06:48:07,141 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/52166ede5574482a91586d16e0f14c87 2024-11-12T06:48:07,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/2f2a552a5dd0467f911fb9e496fbcb7c is 50, key is test_row_0/C:col10/1731394086078/Put/seqid=0 2024-11-12T06:48:07,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741974_1150 (size=12301) 2024-11-12T06:48:07,166 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/2f2a552a5dd0467f911fb9e496fbcb7c 2024-11-12T06:48:07,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/b1067816533b4346acef019a06dcf7e0 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/b1067816533b4346acef019a06dcf7e0 2024-11-12T06:48:07,183 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/b1067816533b4346acef019a06dcf7e0, entries=150, sequenceid=283, filesize=30.5 K 2024-11-12T06:48:07,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/52166ede5574482a91586d16e0f14c87 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/52166ede5574482a91586d16e0f14c87 2024-11-12T06:48:07,196 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/52166ede5574482a91586d16e0f14c87, entries=150, sequenceid=283, filesize=12.0 K 2024-11-12T06:48:07,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/2f2a552a5dd0467f911fb9e496fbcb7c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/2f2a552a5dd0467f911fb9e496fbcb7c 2024-11-12T06:48:07,205 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/2f2a552a5dd0467f911fb9e496fbcb7c, entries=150, sequenceid=283, filesize=12.0 K 2024-11-12T06:48:07,206 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 0a0c05661b31342e3af97a578b0b00fc in 531ms, sequenceid=283, compaction requested=false 2024-11-12T06:48:07,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:07,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:07,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-12T06:48:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-12T06:48:07,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-12T06:48:07,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 838 msec 2024-11-12T06:48:07,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 844 msec 2024-11-12T06:48:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:07,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T06:48:07,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:07,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:07,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:07,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:07,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:07,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:07,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128f69ef26d03e41c7a107c67f0ad465b1_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394087360/Put/seqid=0 2024-11-12T06:48:07,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741975_1151 (size=14994) 2024-11-12T06:48:07,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:07,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394147382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:07,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:07,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:07,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394147387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394147388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:07,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:07,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394147388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:07,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:07,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394147389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:07,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394147497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394147497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:07,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394147497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:07,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394147497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:07,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:07,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T06:48:07,508 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:07,508 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:48:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-12T06:48:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T06:48:07,511 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:48:07,512 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:48:07,512 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:48:07,516 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/35d6a7259d7e4efe907e7d281b1cbde8 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/35d6a7259d7e4efe907e7d281b1cbde8 2024-11-12T06:48:07,527 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/C of 0a0c05661b31342e3af97a578b0b00fc into 35d6a7259d7e4efe907e7d281b1cbde8(size=12.3 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:07,527 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:07,527 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/C, priority=13, startTime=1731394086611; duration=0sec 2024-11-12T06:48:07,527 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:07,527 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:C 2024-11-12T06:48:07,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T06:48:07,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T06:48:07,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:07,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:07,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:07,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:07,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:07,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:07,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:07,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394147707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:07,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:07,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:07,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394147707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:07,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:07,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:07,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394147707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:07,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:07,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394147708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:07,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:07,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:07,781 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:07,785 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128f69ef26d03e41c7a107c67f0ad465b1_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128f69ef26d03e41c7a107c67f0ad465b1_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:07,786 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/44ec13cdb25045f3b038e04d41e1b635, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:07,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/44ec13cdb25045f3b038e04d41e1b635 is 175, key is test_row_0/A:col10/1731394087360/Put/seqid=0 2024-11-12T06:48:07,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741976_1152 (size=39949) 2024-11-12T06:48:07,791 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/44ec13cdb25045f3b038e04d41e1b635 2024-11-12T06:48:07,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/aecb5f98d2544f3d8c4d64b2c82d5286 is 50, key is test_row_0/B:col10/1731394087360/Put/seqid=0 2024-11-12T06:48:07,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741977_1153 (size=12301) 2024-11-12T06:48:07,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T06:48:07,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:07,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:07,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:07,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:07,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:07,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:07,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T06:48:07,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:07,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394147897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:07,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:07,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:07,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T06:48:07,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:07,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:07,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:07,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:07,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:07,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:08,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:08,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394148017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:08,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:08,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:08,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:08,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394148017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:08,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:08,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:08,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:08,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:08,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394148017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:08,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:08,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:08,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394148018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:08,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T06:48:08,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:08,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:08,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:08,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:08,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:08,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T06:48:08,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/aecb5f98d2544f3d8c4d64b2c82d5286 2024-11-12T06:48:08,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/008034b457614f28bfca118cd2e9595c is 50, key is test_row_0/C:col10/1731394087360/Put/seqid=0 2024-11-12T06:48:08,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741978_1154 (size=12301) 2024-11-12T06:48:08,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T06:48:08,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:08,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:08,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:08,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:08,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:08,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T06:48:08,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:08,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:08,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:08,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:08,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:08,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:08,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:08,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394148527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:08,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394148527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:08,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:08,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:08,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394148527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:08,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394148527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:08,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:08,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:08,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:08,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:08,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:08,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:08,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:08,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T06:48:08,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:08,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:08,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:08,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:08,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:08,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:08,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/008034b457614f28bfca118cd2e9595c 2024-11-12T06:48:08,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/44ec13cdb25045f3b038e04d41e1b635 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/44ec13cdb25045f3b038e04d41e1b635 2024-11-12T06:48:08,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/44ec13cdb25045f3b038e04d41e1b635, entries=200, sequenceid=299, filesize=39.0 K 2024-11-12T06:48:08,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/aecb5f98d2544f3d8c4d64b2c82d5286 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/aecb5f98d2544f3d8c4d64b2c82d5286 2024-11-12T06:48:08,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/aecb5f98d2544f3d8c4d64b2c82d5286, entries=150, sequenceid=299, filesize=12.0 K 2024-11-12T06:48:08,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/008034b457614f28bfca118cd2e9595c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/008034b457614f28bfca118cd2e9595c 2024-11-12T06:48:08,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/008034b457614f28bfca118cd2e9595c, entries=150, sequenceid=299, filesize=12.0 K 2024-11-12T06:48:08,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 0a0c05661b31342e3af97a578b0b00fc in 1278ms, sequenceid=299, compaction requested=true 2024-11-12T06:48:08,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:08,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:48:08,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:08,639 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:08,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:48:08,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:08,639 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:08,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:48:08,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:08,640 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:08,640 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102801 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:08,640 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/A is initiating minor compaction (all files) 2024-11-12T06:48:08,640 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/B is initiating minor compaction (all files) 2024-11-12T06:48:08,640 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/A in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:08,640 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/B in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:08,640 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/860331e24d564b50a4ac26400bd2549d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/b1067816533b4346acef019a06dcf7e0, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/44ec13cdb25045f3b038e04d41e1b635] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=100.4 K 2024-11-12T06:48:08,640 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/0a6f5d8b908249f2bfc25fc56037b4a3, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/52166ede5574482a91586d16e0f14c87, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/aecb5f98d2544f3d8c4d64b2c82d5286] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=36.4 K 2024-11-12T06:48:08,640 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:08,641 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. files: [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/860331e24d564b50a4ac26400bd2549d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/b1067816533b4346acef019a06dcf7e0, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/44ec13cdb25045f3b038e04d41e1b635] 2024-11-12T06:48:08,641 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a6f5d8b908249f2bfc25fc56037b4a3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731394085910 2024-11-12T06:48:08,641 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 860331e24d564b50a4ac26400bd2549d, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731394085910 2024-11-12T06:48:08,641 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1067816533b4346acef019a06dcf7e0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731394086074 2024-11-12T06:48:08,641 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 52166ede5574482a91586d16e0f14c87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731394086074 2024-11-12T06:48:08,642 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44ec13cdb25045f3b038e04d41e1b635, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731394086726 2024-11-12T06:48:08,642 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting aecb5f98d2544f3d8c4d64b2c82d5286, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731394086726 2024-11-12T06:48:08,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T06:48:08,648 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:08,650 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#B#compaction#127 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:08,651 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/b668af85ba144f2f85236d2fbf3646e2 is 50, key is test_row_0/B:col10/1731394087360/Put/seqid=0 2024-11-12T06:48:08,659 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111266ceb89795434492ad576d1e9a6518e0_0a0c05661b31342e3af97a578b0b00fc store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:08,662 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111266ceb89795434492ad576d1e9a6518e0_0a0c05661b31342e3af97a578b0b00fc, store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:08,662 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111266ceb89795434492ad576d1e9a6518e0_0a0c05661b31342e3af97a578b0b00fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:08,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741979_1155 (size=12745) 2024-11-12T06:48:08,674 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/b668af85ba144f2f85236d2fbf3646e2 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/b668af85ba144f2f85236d2fbf3646e2 2024-11-12T06:48:08,681 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/B of 0a0c05661b31342e3af97a578b0b00fc into b668af85ba144f2f85236d2fbf3646e2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:08,681 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:08,681 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/B, priority=13, startTime=1731394088639; duration=0sec 2024-11-12T06:48:08,681 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:08,681 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:B 2024-11-12T06:48:08,681 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:08,683 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:08,683 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/C is initiating minor compaction (all files) 2024-11-12T06:48:08,683 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/C in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:08,683 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/35d6a7259d7e4efe907e7d281b1cbde8, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/2f2a552a5dd0467f911fb9e496fbcb7c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/008034b457614f28bfca118cd2e9595c] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=36.4 K 2024-11-12T06:48:08,684 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 35d6a7259d7e4efe907e7d281b1cbde8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731394085910 2024-11-12T06:48:08,684 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f2a552a5dd0467f911fb9e496fbcb7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731394086074 2024-11-12T06:48:08,685 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 008034b457614f28bfca118cd2e9595c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731394086726 2024-11-12T06:48:08,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741980_1156 (size=4469) 2024-11-12T06:48:08,695 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#C#compaction#128 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:08,696 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/8905faaf37f44e968b5c79d718abd351 is 50, key is test_row_0/C:col10/1731394087360/Put/seqid=0 2024-11-12T06:48:08,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741981_1157 (size=12745) 2024-11-12T06:48:08,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T06:48:08,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:08,736 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T06:48:08,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:08,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:08,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:08,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:08,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:08,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:08,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120d73253385144519b63a92e7b0a81dad_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394087379/Put/seqid=0 2024-11-12T06:48:08,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741982_1158 (size=12454) 2024-11-12T06:48:08,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:08,755 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120d73253385144519b63a92e7b0a81dad_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120d73253385144519b63a92e7b0a81dad_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:08,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/a78f17ef7122442885195dda8c9f3e3f, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:08,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/a78f17ef7122442885195dda8c9f3e3f is 175, key is test_row_0/A:col10/1731394087379/Put/seqid=0 2024-11-12T06:48:08,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741983_1159 (size=31255) 2024-11-12T06:48:08,768 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/a78f17ef7122442885195dda8c9f3e3f 2024-11-12T06:48:08,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/c355430b825e46d4b1216e5fec29fcf4 is 50, key is test_row_0/B:col10/1731394087379/Put/seqid=0 2024-11-12T06:48:08,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741984_1160 (size=12301) 2024-11-12T06:48:08,787 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/c355430b825e46d4b1216e5fec29fcf4 2024-11-12T06:48:08,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/631f48b40b3949df81205021ab7cf046 is 50, key is test_row_0/C:col10/1731394087379/Put/seqid=0 2024-11-12T06:48:08,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741985_1161 (size=12301) 2024-11-12T06:48:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:08,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:08,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394148957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:09,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394149067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:09,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:09,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:09,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:09,092 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#A#compaction#126 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:09,093 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/196bbfb1d82c459aa35260a07a2b4bd0 is 175, key is test_row_0/A:col10/1731394087360/Put/seqid=0 2024-11-12T06:48:09,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741986_1162 (size=31699) 2024-11-12T06:48:09,107 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/196bbfb1d82c459aa35260a07a2b4bd0 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/196bbfb1d82c459aa35260a07a2b4bd0 2024-11-12T06:48:09,109 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/8905faaf37f44e968b5c79d718abd351 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/8905faaf37f44e968b5c79d718abd351 2024-11-12T06:48:09,114 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/A of 0a0c05661b31342e3af97a578b0b00fc into 196bbfb1d82c459aa35260a07a2b4bd0(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:09,114 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:09,114 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/A, priority=13, startTime=1731394088639; duration=0sec 2024-11-12T06:48:09,114 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:09,115 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:A 2024-11-12T06:48:09,117 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/C of 0a0c05661b31342e3af97a578b0b00fc into 8905faaf37f44e968b5c79d718abd351(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:09,117 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:09,117 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/C, priority=13, startTime=1731394088639; duration=0sec 2024-11-12T06:48:09,117 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:09,117 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:C 2024-11-12T06:48:09,213 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/631f48b40b3949df81205021ab7cf046 2024-11-12T06:48:09,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/a78f17ef7122442885195dda8c9f3e3f as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/a78f17ef7122442885195dda8c9f3e3f 2024-11-12T06:48:09,223 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/a78f17ef7122442885195dda8c9f3e3f, entries=150, sequenceid=319, filesize=30.5 K 2024-11-12T06:48:09,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/c355430b825e46d4b1216e5fec29fcf4 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/c355430b825e46d4b1216e5fec29fcf4 2024-11-12T06:48:09,228 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/c355430b825e46d4b1216e5fec29fcf4, entries=150, sequenceid=319, filesize=12.0 K 2024-11-12T06:48:09,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/631f48b40b3949df81205021ab7cf046 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/631f48b40b3949df81205021ab7cf046 2024-11-12T06:48:09,233 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/631f48b40b3949df81205021ab7cf046, entries=150, sequenceid=319, filesize=12.0 K 2024-11-12T06:48:09,235 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 0a0c05661b31342e3af97a578b0b00fc in 498ms, sequenceid=319, compaction requested=false 2024-11-12T06:48:09,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:09,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:09,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-12T06:48:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-12T06:48:09,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-12T06:48:09,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7240 sec 2024-11-12T06:48:09,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.7300 sec 2024-11-12T06:48:09,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T06:48:09,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:09,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:09,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:09,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:09,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:09,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:09,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123cc750a8b54843f99e2be1af6e99928d_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394088941/Put/seqid=0 2024-11-12T06:48:09,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741987_1163 (size=14994) 2024-11-12T06:48:09,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394149336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:09,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394149447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:09,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394149547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:09,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394149547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:09,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394149547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:09,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:09,550 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_0' in 0a0c05661b31342e3af97a578b0b00fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:48:09,550 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_1' in 0a0c05661b31342e3af97a578b0b00fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:48:09,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:09,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:09,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:09,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:09,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:09,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:09,550 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_2' in 0a0c05661b31342e3af97a578b0b00fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:48:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394149547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:09,551 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_0' in 0a0c05661b31342e3af97a578b0b00fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:48:09,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:09,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:09,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T06:48:09,657 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:09,658 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:48:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-12T06:48:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T06:48:09,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394149657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:09,660 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:48:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:09,660 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:48:09,660 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:48:09,714 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:09,718 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123cc750a8b54843f99e2be1af6e99928d_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123cc750a8b54843f99e2be1af6e99928d_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:09,719 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/a2ffe4ee50114f37b013a34a7acec6d4, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:09,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/a2ffe4ee50114f37b013a34a7acec6d4 is 175, key is test_row_0/A:col10/1731394088941/Put/seqid=0 2024-11-12T06:48:09,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741988_1164 (size=39949) 2024-11-12T06:48:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T06:48:09,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T06:48:09,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:09,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:09,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:09,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:09,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:09,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:09,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T06:48:09,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:09,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:09,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:09,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:09,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:09,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394149967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:09,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T06:48:10,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T06:48:10,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:10,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:10,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:10,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:10,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:10,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:10,125 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/a2ffe4ee50114f37b013a34a7acec6d4 2024-11-12T06:48:10,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/afba092782554d72ab7dbaeb6942252c is 50, key is test_row_0/B:col10/1731394088941/Put/seqid=0 2024-11-12T06:48:10,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741989_1165 (size=12301) 2024-11-12T06:48:10,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/afba092782554d72ab7dbaeb6942252c 2024-11-12T06:48:10,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/23c4337e156e4b2bafb4f22d9400360a is 50, key is test_row_0/C:col10/1731394088941/Put/seqid=0 2024-11-12T06:48:10,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741990_1166 (size=12301) 2024-11-12T06:48:10,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T06:48:10,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:10,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:10,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:10,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:10,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T06:48:10,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T06:48:10,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:10,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:10,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:10,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:10,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:10,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:10,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:10,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394150477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:10,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/23c4337e156e4b2bafb4f22d9400360a 2024-11-12T06:48:10,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/a2ffe4ee50114f37b013a34a7acec6d4 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/a2ffe4ee50114f37b013a34a7acec6d4 2024-11-12T06:48:10,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/a2ffe4ee50114f37b013a34a7acec6d4, entries=200, sequenceid=336, filesize=39.0 K 2024-11-12T06:48:10,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/afba092782554d72ab7dbaeb6942252c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/afba092782554d72ab7dbaeb6942252c 2024-11-12T06:48:10,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/afba092782554d72ab7dbaeb6942252c, entries=150, sequenceid=336, filesize=12.0 K 2024-11-12T06:48:10,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/23c4337e156e4b2bafb4f22d9400360a as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/23c4337e156e4b2bafb4f22d9400360a 2024-11-12T06:48:10,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/23c4337e156e4b2bafb4f22d9400360a, entries=150, sequenceid=336, filesize=12.0 K 2024-11-12T06:48:10,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 0a0c05661b31342e3af97a578b0b00fc in 1293ms, sequenceid=336, compaction requested=true 2024-11-12T06:48:10,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:10,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:48:10,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:10,572 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:10,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:48:10,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:10,572 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:10,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:48:10,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:10,573 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:10,573 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:10,573 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/A is initiating minor compaction (all files) 2024-11-12T06:48:10,573 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/B is initiating minor compaction (all files) 2024-11-12T06:48:10,573 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/A in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:10,573 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/B in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:10,573 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/196bbfb1d82c459aa35260a07a2b4bd0, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/a78f17ef7122442885195dda8c9f3e3f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/a2ffe4ee50114f37b013a34a7acec6d4] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=100.5 K 2024-11-12T06:48:10,573 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/b668af85ba144f2f85236d2fbf3646e2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/c355430b825e46d4b1216e5fec29fcf4, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/afba092782554d72ab7dbaeb6942252c] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=36.5 K 2024-11-12T06:48:10,573 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:10,573 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. files: [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/196bbfb1d82c459aa35260a07a2b4bd0, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/a78f17ef7122442885195dda8c9f3e3f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/a2ffe4ee50114f37b013a34a7acec6d4] 2024-11-12T06:48:10,574 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 196bbfb1d82c459aa35260a07a2b4bd0, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731394086726 2024-11-12T06:48:10,574 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting b668af85ba144f2f85236d2fbf3646e2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731394086726 2024-11-12T06:48:10,574 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting a78f17ef7122442885195dda8c9f3e3f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731394087379 2024-11-12T06:48:10,574 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting c355430b825e46d4b1216e5fec29fcf4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731394087379 2024-11-12T06:48:10,574 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2ffe4ee50114f37b013a34a7acec6d4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731394088941 2024-11-12T06:48:10,574 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting afba092782554d72ab7dbaeb6942252c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731394088941 2024-11-12T06:48:10,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T06:48:10,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:10,579 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T06:48:10,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:10,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:10,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:10,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:10,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:10,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:10,583 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:10,592 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112e0f66b4fdc004ea4a6c0b3de6ec55768_0a0c05661b31342e3af97a578b0b00fc store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:10,593 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#B#compaction#136 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:10,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112bac0114268b244b5b68b70e3bfa3e19d_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394089332/Put/seqid=0 2024-11-12T06:48:10,593 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/8839b22eec534b22a67c0debd43dac0a is 50, key is test_row_0/B:col10/1731394088941/Put/seqid=0 2024-11-12T06:48:10,594 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112e0f66b4fdc004ea4a6c0b3de6ec55768_0a0c05661b31342e3af97a578b0b00fc, store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:10,594 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e0f66b4fdc004ea4a6c0b3de6ec55768_0a0c05661b31342e3af97a578b0b00fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:10,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741991_1167 (size=12847) 2024-11-12T06:48:10,605 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/8839b22eec534b22a67c0debd43dac0a as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/8839b22eec534b22a67c0debd43dac0a 2024-11-12T06:48:10,610 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/B of 0a0c05661b31342e3af97a578b0b00fc into 8839b22eec534b22a67c0debd43dac0a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:10,610 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:10,611 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/B, priority=13, startTime=1731394090572; duration=0sec 2024-11-12T06:48:10,611 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:10,611 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:B 2024-11-12T06:48:10,611 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:10,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741993_1169 (size=4469) 2024-11-12T06:48:10,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741992_1168 (size=12454) 2024-11-12T06:48:10,613 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:10,613 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/C is initiating minor compaction (all files) 2024-11-12T06:48:10,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:10,613 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/C in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:10,614 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/8905faaf37f44e968b5c79d718abd351, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/631f48b40b3949df81205021ab7cf046, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/23c4337e156e4b2bafb4f22d9400360a] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=36.5 K 2024-11-12T06:48:10,614 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 8905faaf37f44e968b5c79d718abd351, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731394086726 2024-11-12T06:48:10,616 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 631f48b40b3949df81205021ab7cf046, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731394087379 2024-11-12T06:48:10,622 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#A#compaction#135 average throughput is 0.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:10,623 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/7df65ee81c79453b92d913b2c5fefe00 is 175, key is test_row_0/A:col10/1731394088941/Put/seqid=0 2024-11-12T06:48:10,623 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 23c4337e156e4b2bafb4f22d9400360a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731394088941 2024-11-12T06:48:10,625 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112bac0114268b244b5b68b70e3bfa3e19d_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112bac0114268b244b5b68b70e3bfa3e19d_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:10,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/9373f530d3674cabaa3c99c5c9683513, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:10,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/9373f530d3674cabaa3c99c5c9683513 is 175, key is test_row_0/A:col10/1731394089332/Put/seqid=0 2024-11-12T06:48:10,644 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#C#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:10,644 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/64ac88ab754a4fc3abee6696c7cb86fa is 50, key is test_row_0/C:col10/1731394088941/Put/seqid=0 2024-11-12T06:48:10,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741996_1172 (size=12847) 2024-11-12T06:48:10,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741995_1171 (size=31255) 2024-11-12T06:48:10,657 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/9373f530d3674cabaa3c99c5c9683513 2024-11-12T06:48:10,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741994_1170 (size=31801) 2024-11-12T06:48:10,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/a56da07cd74c45bbae2aa0ab6c015350 is 50, key is test_row_0/B:col10/1731394089332/Put/seqid=0 2024-11-12T06:48:10,672 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/64ac88ab754a4fc3abee6696c7cb86fa as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/64ac88ab754a4fc3abee6696c7cb86fa 2024-11-12T06:48:10,675 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/7df65ee81c79453b92d913b2c5fefe00 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/7df65ee81c79453b92d913b2c5fefe00 2024-11-12T06:48:10,678 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/C of 0a0c05661b31342e3af97a578b0b00fc into 64ac88ab754a4fc3abee6696c7cb86fa(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:10,678 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:10,679 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/C, priority=13, startTime=1731394090572; duration=0sec 2024-11-12T06:48:10,681 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:10,681 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:C 2024-11-12T06:48:10,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741997_1173 (size=12301) 2024-11-12T06:48:10,686 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/A of 0a0c05661b31342e3af97a578b0b00fc into 7df65ee81c79453b92d913b2c5fefe00(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:10,686 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:10,686 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/A, priority=13, startTime=1731394090572; duration=0sec 2024-11-12T06:48:10,686 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:10,686 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:A 2024-11-12T06:48:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T06:48:11,085 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/a56da07cd74c45bbae2aa0ab6c015350 2024-11-12T06:48:11,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/379c0c24b02c425b8f2b48ca4093ca5a is 50, key is test_row_0/C:col10/1731394089332/Put/seqid=0 2024-11-12T06:48:11,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741998_1174 (size=12301) 2024-11-12T06:48:11,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:11,506 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/379c0c24b02c425b8f2b48ca4093ca5a 2024-11-12T06:48:11,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/9373f530d3674cabaa3c99c5c9683513 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/9373f530d3674cabaa3c99c5c9683513 2024-11-12T06:48:11,520 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/9373f530d3674cabaa3c99c5c9683513, entries=150, sequenceid=353, filesize=30.5 K 2024-11-12T06:48:11,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/a56da07cd74c45bbae2aa0ab6c015350 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/a56da07cd74c45bbae2aa0ab6c015350 2024-11-12T06:48:11,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394151524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:11,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:11,527 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/a56da07cd74c45bbae2aa0ab6c015350, entries=150, sequenceid=353, filesize=12.0 K 2024-11-12T06:48:11,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/379c0c24b02c425b8f2b48ca4093ca5a as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/379c0c24b02c425b8f2b48ca4093ca5a 2024-11-12T06:48:11,534 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/379c0c24b02c425b8f2b48ca4093ca5a, entries=150, sequenceid=353, filesize=12.0 K 2024-11-12T06:48:11,535 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 0a0c05661b31342e3af97a578b0b00fc in 956ms, sequenceid=353, compaction requested=false 2024-11-12T06:48:11,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:11,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:11,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-12T06:48:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-12T06:48:11,538 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-12T06:48:11,538 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8760 sec 2024-11-12T06:48:11,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.8800 sec 2024-11-12T06:48:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:11,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T06:48:11,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:11,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:11,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:11,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:11,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:11,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:11,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a568eedc6c464b83903c8a78044ee821_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394091560/Put/seqid=0 2024-11-12T06:48:11,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741999_1175 (size=12454) 2024-11-12T06:48:11,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:11,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394151588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:11,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394151589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:11,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:11,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394151591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:11,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:11,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:11,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394151592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:11,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:11,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:11,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394151627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:11,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394151697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:11,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394151697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:11,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:11,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394151707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394151707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T06:48:11,808 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:11,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:48:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-12T06:48:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T06:48:11,810 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:48:11,810 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:48:11,811 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:48:11,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394151837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:11,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:11,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394151907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:11,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394151907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:11,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:11,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T06:48:11,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394151917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:11,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394151917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:11,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:11,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:11,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:11,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T06:48:11,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:11,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:11,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:11,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:11,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:11,990 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:11,994 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a568eedc6c464b83903c8a78044ee821_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a568eedc6c464b83903c8a78044ee821_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:11,995 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/dad191a7b1b54aaab895d1a1be49192b, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:11,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/dad191a7b1b54aaab895d1a1be49192b is 175, key is test_row_0/A:col10/1731394091560/Put/seqid=0 2024-11-12T06:48:12,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742000_1176 (size=31255) 2024-11-12T06:48:12,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T06:48:12,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:12,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:12,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:12,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:12,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T06:48:12,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394152147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:12,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:12,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:12,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:12,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:12,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394152227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:12,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:12,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:12,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:12,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394152227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:12,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:12,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:12,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:12,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:12,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:12,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394152227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:12,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:12,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:12,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:12,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:12,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394152227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:12,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:12,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:12,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:12,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T06:48:12,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:12,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:12,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:12,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:12,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:12,404 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=373, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/dad191a7b1b54aaab895d1a1be49192b 2024-11-12T06:48:12,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/23c52894a6e848afb44fc99241738747 is 50, key is test_row_0/B:col10/1731394091560/Put/seqid=0 2024-11-12T06:48:12,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742001_1177 (size=12301) 2024-11-12T06:48:12,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/23c52894a6e848afb44fc99241738747 2024-11-12T06:48:12,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T06:48:12,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:12,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:12,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:12,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:12,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:12,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/1fd5c5999e7846d583cf38e2e47a89f2 is 50, key is test_row_0/C:col10/1731394091560/Put/seqid=0 2024-11-12T06:48:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:12,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742002_1178 (size=12301) 2024-11-12T06:48:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T06:48:12,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T06:48:12,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:12,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:12,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:12,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:12,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:12,603 INFO [master/9f0beee3d6b6:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-12T06:48:12,603 INFO [master/9f0beee3d6b6:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-12T06:48:12,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394152657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:12,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:12,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T06:48:12,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:12,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:12,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:12,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:12,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:12,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:12,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:12,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394152737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:12,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394152737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:12,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:12,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:12,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:12,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394152738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:12,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:12,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394152738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:12,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/1fd5c5999e7846d583cf38e2e47a89f2 2024-11-12T06:48:12,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/dad191a7b1b54aaab895d1a1be49192b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/dad191a7b1b54aaab895d1a1be49192b 2024-11-12T06:48:12,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/dad191a7b1b54aaab895d1a1be49192b, entries=150, sequenceid=373, filesize=30.5 K 2024-11-12T06:48:12,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/23c52894a6e848afb44fc99241738747 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/23c52894a6e848afb44fc99241738747 2024-11-12T06:48:12,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/23c52894a6e848afb44fc99241738747, entries=150, sequenceid=373, filesize=12.0 K 2024-11-12T06:48:12,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/1fd5c5999e7846d583cf38e2e47a89f2 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/1fd5c5999e7846d583cf38e2e47a89f2 2024-11-12T06:48:12,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/1fd5c5999e7846d583cf38e2e47a89f2, entries=150, sequenceid=373, filesize=12.0 K 2024-11-12T06:48:12,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 0a0c05661b31342e3af97a578b0b00fc in 1286ms, sequenceid=373, compaction requested=true 2024-11-12T06:48:12,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:12,847 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:12,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:48:12,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:12,849 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:12,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:48:12,849 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:12,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:12,849 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/A is initiating minor compaction (all files) 2024-11-12T06:48:12,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:48:12,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:12,849 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/A in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:12,849 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/7df65ee81c79453b92d913b2c5fefe00, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/9373f530d3674cabaa3c99c5c9683513, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/dad191a7b1b54aaab895d1a1be49192b] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=92.1 K 2024-11-12T06:48:12,849 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:12,849 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. files: [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/7df65ee81c79453b92d913b2c5fefe00, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/9373f530d3674cabaa3c99c5c9683513, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/dad191a7b1b54aaab895d1a1be49192b] 2024-11-12T06:48:12,850 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7df65ee81c79453b92d913b2c5fefe00, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731394088941 2024-11-12T06:48:12,850 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9373f530d3674cabaa3c99c5c9683513, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731394089327 2024-11-12T06:48:12,850 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:12,850 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting dad191a7b1b54aaab895d1a1be49192b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731394091508 2024-11-12T06:48:12,850 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/B is initiating minor compaction (all files) 2024-11-12T06:48:12,850 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/B in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:12,850 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/8839b22eec534b22a67c0debd43dac0a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/a56da07cd74c45bbae2aa0ab6c015350, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/23c52894a6e848afb44fc99241738747] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=36.6 K 2024-11-12T06:48:12,851 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 8839b22eec534b22a67c0debd43dac0a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731394088941 2024-11-12T06:48:12,851 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting a56da07cd74c45bbae2aa0ab6c015350, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731394089327 2024-11-12T06:48:12,852 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 23c52894a6e848afb44fc99241738747, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731394091508 2024-11-12T06:48:12,858 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:12,861 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#B#compaction#145 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:12,861 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/f8f682c31e20472990321783ce3ec337 is 50, key is test_row_0/B:col10/1731394091560/Put/seqid=0 2024-11-12T06:48:12,865 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112455fb593c2184c66af8c440d59882384_0a0c05661b31342e3af97a578b0b00fc store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:12,868 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112455fb593c2184c66af8c440d59882384_0a0c05661b31342e3af97a578b0b00fc, store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:12,868 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112455fb593c2184c66af8c440d59882384_0a0c05661b31342e3af97a578b0b00fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:12,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742003_1179 (size=12949) 2024-11-12T06:48:12,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742004_1180 (size=4469) 2024-11-12T06:48:12,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T06:48:12,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:12,881 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T06:48:12,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:12,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:12,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:12,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:12,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:12,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:12,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a34ab5d6fd1e43f39510ba722c8d3d11_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394091578/Put/seqid=0 2024-11-12T06:48:12,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742005_1181 (size=12454) 2024-11-12T06:48:12,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:12,898 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a34ab5d6fd1e43f39510ba722c8d3d11_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a34ab5d6fd1e43f39510ba722c8d3d11_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:12,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/57e02573e8a7464596752d9aa52971ba, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:12,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/57e02573e8a7464596752d9aa52971ba is 175, key is test_row_0/A:col10/1731394091578/Put/seqid=0 2024-11-12T06:48:12,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742006_1182 (size=31255) 2024-11-12T06:48:12,919 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=389, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/57e02573e8a7464596752d9aa52971ba 2024-11-12T06:48:12,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/03f154e9cdcd421693f52deabc2959a2 is 50, key is test_row_0/B:col10/1731394091578/Put/seqid=0 2024-11-12T06:48:12,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742007_1183 (size=12301) 2024-11-12T06:48:12,942 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/03f154e9cdcd421693f52deabc2959a2 2024-11-12T06:48:12,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T06:48:12,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/397a7eb755154f3ea80d24dfe26a976e is 50, key is test_row_0/C:col10/1731394091578/Put/seqid=0 2024-11-12T06:48:12,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742008_1184 (size=12301) 2024-11-12T06:48:13,282 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#A#compaction#144 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:13,282 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/73a5d86dffe44ef6a8d20bdb1bf63567 is 175, key is test_row_0/A:col10/1731394091560/Put/seqid=0 2024-11-12T06:48:13,286 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/f8f682c31e20472990321783ce3ec337 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/f8f682c31e20472990321783ce3ec337 2024-11-12T06:48:13,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742009_1185 (size=31903) 2024-11-12T06:48:13,291 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/B of 0a0c05661b31342e3af97a578b0b00fc into f8f682c31e20472990321783ce3ec337(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:13,291 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:13,291 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/B, priority=13, startTime=1731394092849; duration=0sec 2024-11-12T06:48:13,291 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:13,291 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:B 2024-11-12T06:48:13,291 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:13,292 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:13,292 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/C is initiating minor compaction (all files) 2024-11-12T06:48:13,293 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/C in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:13,293 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/64ac88ab754a4fc3abee6696c7cb86fa, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/379c0c24b02c425b8f2b48ca4093ca5a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/1fd5c5999e7846d583cf38e2e47a89f2] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=36.6 K 2024-11-12T06:48:13,293 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 64ac88ab754a4fc3abee6696c7cb86fa, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731394088941 2024-11-12T06:48:13,293 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 379c0c24b02c425b8f2b48ca4093ca5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731394089327 2024-11-12T06:48:13,294 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fd5c5999e7846d583cf38e2e47a89f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731394091508 2024-11-12T06:48:13,301 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#C#compaction#149 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:13,302 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/d963aeeb48fb4f238d5b51dd84e8445c is 50, key is test_row_0/C:col10/1731394091560/Put/seqid=0 2024-11-12T06:48:13,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742010_1186 (size=12949) 2024-11-12T06:48:13,361 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/397a7eb755154f3ea80d24dfe26a976e 2024-11-12T06:48:13,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/57e02573e8a7464596752d9aa52971ba as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/57e02573e8a7464596752d9aa52971ba 2024-11-12T06:48:13,371 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/57e02573e8a7464596752d9aa52971ba, entries=150, sequenceid=389, filesize=30.5 K 2024-11-12T06:48:13,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/03f154e9cdcd421693f52deabc2959a2 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/03f154e9cdcd421693f52deabc2959a2 2024-11-12T06:48:13,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,376 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/03f154e9cdcd421693f52deabc2959a2, entries=150, sequenceid=389, filesize=12.0 K 2024-11-12T06:48:13,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/397a7eb755154f3ea80d24dfe26a976e as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/397a7eb755154f3ea80d24dfe26a976e 2024-11-12T06:48:13,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,382 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/397a7eb755154f3ea80d24dfe26a976e, entries=150, sequenceid=389, filesize=12.0 K 2024-11-12T06:48:13,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,383 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 0a0c05661b31342e3af97a578b0b00fc in 502ms, sequenceid=389, compaction requested=false 2024-11-12T06:48:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:13,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:13,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-12T06:48:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-12T06:48:13,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-12T06:48:13,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5750 sec 2024-11-12T06:48:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.5810 sec 2024-11-12T06:48:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,697 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/73a5d86dffe44ef6a8d20bdb1bf63567 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/73a5d86dffe44ef6a8d20bdb1bf63567 2024-11-12T06:48:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,703 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/A of 0a0c05661b31342e3af97a578b0b00fc into 73a5d86dffe44ef6a8d20bdb1bf63567(size=31.2 K), total size for store is 61.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:13,703 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,703 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/A, priority=13, startTime=1731394092847; duration=0sec 2024-11-12T06:48:13,703 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:13,703 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:A 2024-11-12T06:48:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,712 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/d963aeeb48fb4f238d5b51dd84e8445c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/d963aeeb48fb4f238d5b51dd84e8445c 2024-11-12T06:48:13,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,718 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/C of 0a0c05661b31342e3af97a578b0b00fc into d963aeeb48fb4f238d5b51dd84e8445c(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:13,718 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:13,718 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/C, priority=13, startTime=1731394092849; duration=0sec 2024-11-12T06:48:13,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,718 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:13,718 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:C 2024-11-12T06:48:13,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:13,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T06:48:13,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:13,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:13,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:13,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:13,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:13,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:13,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124316cd56fe4a472a913c2345bbfbc700_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394093726/Put/seqid=0 2024-11-12T06:48:13,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:13,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742011_1187 (size=22618) 2024-11-12T06:48:13,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394093781, max timestamp=1731394093799Num uniques 859; 2024-11-12T06:48:13,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394093781, max timestamp=1731394093799Num uniques 859; 2024-11-12T06:48:13,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:48:13,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:48:13,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394093781, max timestamp=1731394093799Num uniques 859; 2024-11-12T06:48:13,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:48:13,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:13,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394153825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:13,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:13,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394153825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:13,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:13,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:13,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:13,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:13,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394153827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:13,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:13,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:13,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:13,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:13,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:13,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:13,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:13,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394153829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:13,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:13,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:13,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:13,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:13,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394153830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:13,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:13,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:13,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:13,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394153947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:13,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:13,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:13,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394153947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:13,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:13,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:13,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:13,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:13,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:13,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394153948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:13,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:13,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:13,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394153948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:13,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394153948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:13,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:13,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:13,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:13,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:13,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:13,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:13,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:13,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:13,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:13,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T06:48:13,967 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:13,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:48:13,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-12T06:48:13,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T06:48:13,970 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:48:13,970 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:48:13,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:48:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T06:48:14,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T06:48:14,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:14,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:14,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:14,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:14,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:14,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:14,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394154157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:14,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:14,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394154157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:14,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:14,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394154167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:14,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394154167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:14,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394154167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:14,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:14,193 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:14,200 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124316cd56fe4a472a913c2345bbfbc700_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124316cd56fe4a472a913c2345bbfbc700_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:14,205 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/f6ac5e376de04292925134991d01df4f, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:14,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/f6ac5e376de04292925134991d01df4f is 175, key is test_row_0/A:col10/1731394093726/Put/seqid=0 2024-11-12T06:48:14,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742012_1188 (size=66023) 2024-11-12T06:48:14,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T06:48:14,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:14,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:14,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:14,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:14,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T06:48:14,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T06:48:14,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:14,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:14,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:14,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:14,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:14,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394154467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:14,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:14,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394154467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:14,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:14,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:14,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394154477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:14,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:14,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:14,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394154477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:14,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:14,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:14,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394154477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:14,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:14,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T06:48:14,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:14,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:14,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:14,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:14,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:14,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T06:48:14,610 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=402, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/f6ac5e376de04292925134991d01df4f 2024-11-12T06:48:14,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/f4b5a3c225b74277bb4d9fc8c9265088 is 50, key is test_row_0/B:col10/1731394093726/Put/seqid=0 2024-11-12T06:48:14,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742013_1189 (size=12301) 2024-11-12T06:48:14,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T06:48:14,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:14,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:14,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:14,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:14,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:14,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T06:48:14,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:14,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:14,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:14,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:14,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:14,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394154977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:14,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:14,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394154977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:14,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:14,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:14,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394154987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394154987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:14,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394154987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:15,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/f4b5a3c225b74277bb4d9fc8c9265088 2024-11-12T06:48:15,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/b6d52d4f6fbf484c80e3f927cba11197 is 50, key is test_row_0/C:col10/1731394093726/Put/seqid=0 2024-11-12T06:48:15,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742014_1190 (size=12301) 2024-11-12T06:48:15,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T06:48:15,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:15,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:15,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:15,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:15,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:15,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T06:48:15,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T06:48:15,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:15,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:15,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:15,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:15,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:15,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T06:48:15,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:15,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:15,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:15,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:15,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:15,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/b6d52d4f6fbf484c80e3f927cba11197 2024-11-12T06:48:15,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/f6ac5e376de04292925134991d01df4f as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/f6ac5e376de04292925134991d01df4f 2024-11-12T06:48:15,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/f6ac5e376de04292925134991d01df4f, entries=350, sequenceid=402, filesize=64.5 K 2024-11-12T06:48:15,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/f4b5a3c225b74277bb4d9fc8c9265088 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/f4b5a3c225b74277bb4d9fc8c9265088 2024-11-12T06:48:15,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/f4b5a3c225b74277bb4d9fc8c9265088, entries=150, sequenceid=402, filesize=12.0 K 2024-11-12T06:48:15,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/b6d52d4f6fbf484c80e3f927cba11197 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/b6d52d4f6fbf484c80e3f927cba11197 2024-11-12T06:48:15,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/b6d52d4f6fbf484c80e3f927cba11197, entries=150, sequenceid=402, filesize=12.0 K 2024-11-12T06:48:15,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0a0c05661b31342e3af97a578b0b00fc in 1704ms, sequenceid=402, compaction requested=true 2024-11-12T06:48:15,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:15,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:48:15,456 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:15,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:15,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:48:15,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:15,456 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:15,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:48:15,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:15,457 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129181 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:15,457 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:15,458 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/A is initiating minor compaction (all files) 2024-11-12T06:48:15,458 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/B is initiating minor compaction (all files) 2024-11-12T06:48:15,458 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/A in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:15,458 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/B in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:15,458 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/73a5d86dffe44ef6a8d20bdb1bf63567, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/57e02573e8a7464596752d9aa52971ba, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/f6ac5e376de04292925134991d01df4f] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=126.2 K 2024-11-12T06:48:15,458 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/f8f682c31e20472990321783ce3ec337, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/03f154e9cdcd421693f52deabc2959a2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/f4b5a3c225b74277bb4d9fc8c9265088] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=36.7 K 2024-11-12T06:48:15,458 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:15,458 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. files: [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/73a5d86dffe44ef6a8d20bdb1bf63567, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/57e02573e8a7464596752d9aa52971ba, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/f6ac5e376de04292925134991d01df4f] 2024-11-12T06:48:15,458 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting f8f682c31e20472990321783ce3ec337, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731394091508 2024-11-12T06:48:15,458 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73a5d86dffe44ef6a8d20bdb1bf63567, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731394091508 2024-11-12T06:48:15,458 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 03f154e9cdcd421693f52deabc2959a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731394091578 2024-11-12T06:48:15,459 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57e02573e8a7464596752d9aa52971ba, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731394091578 2024-11-12T06:48:15,459 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting f4b5a3c225b74277bb4d9fc8c9265088, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731394093726 2024-11-12T06:48:15,459 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6ac5e376de04292925134991d01df4f, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731394093678 2024-11-12T06:48:15,464 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:15,466 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#B#compaction#154 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:15,467 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/d8890594924e486698096810d7634217 is 50, key is test_row_0/B:col10/1731394093726/Put/seqid=0 2024-11-12T06:48:15,468 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112b409d8ceaaff4068a83444456d434abb_0a0c05661b31342e3af97a578b0b00fc store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:15,471 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112b409d8ceaaff4068a83444456d434abb_0a0c05661b31342e3af97a578b0b00fc, store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:15,471 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b409d8ceaaff4068a83444456d434abb_0a0c05661b31342e3af97a578b0b00fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:15,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742016_1192 (size=4469) 2024-11-12T06:48:15,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742015_1191 (size=13051) 2024-11-12T06:48:15,484 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#A#compaction#153 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:15,485 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/43a75e5252a94e8490bede971f744f41 is 175, key is test_row_0/A:col10/1731394093726/Put/seqid=0 2024-11-12T06:48:15,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742017_1193 (size=32005) 2024-11-12T06:48:15,495 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/43a75e5252a94e8490bede971f744f41 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/43a75e5252a94e8490bede971f744f41 2024-11-12T06:48:15,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T06:48:15,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:15,500 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-12T06:48:15,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:15,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:15,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:15,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:15,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:15,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:15,503 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/A of 0a0c05661b31342e3af97a578b0b00fc into 43a75e5252a94e8490bede971f744f41(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:15,503 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:15,503 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/A, priority=13, startTime=1731394095456; duration=0sec 2024-11-12T06:48:15,503 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:15,503 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:A 2024-11-12T06:48:15,503 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:15,505 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:15,505 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/C is initiating minor compaction (all files) 2024-11-12T06:48:15,505 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/C in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:15,505 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/d963aeeb48fb4f238d5b51dd84e8445c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/397a7eb755154f3ea80d24dfe26a976e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/b6d52d4f6fbf484c80e3f927cba11197] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=36.7 K 2024-11-12T06:48:15,505 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting d963aeeb48fb4f238d5b51dd84e8445c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731394091508 2024-11-12T06:48:15,506 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 397a7eb755154f3ea80d24dfe26a976e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731394091578 2024-11-12T06:48:15,506 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6d52d4f6fbf484c80e3f927cba11197, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731394093726 2024-11-12T06:48:15,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124652878bbdec49efacdcc16c1040ffac_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394093823/Put/seqid=0 2024-11-12T06:48:15,519 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#C#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:15,519 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/4baca241e6f646eb877f64ef163e9eec is 50, key is test_row_0/C:col10/1731394093726/Put/seqid=0 2024-11-12T06:48:15,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742018_1194 (size=12454) 2024-11-12T06:48:15,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,536 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124652878bbdec49efacdcc16c1040ffac_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124652878bbdec49efacdcc16c1040ffac_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:15,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/3067afcb0c2b4ea19af799f1519b2d5f, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:15,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/3067afcb0c2b4ea19af799f1519b2d5f is 175, key is test_row_0/A:col10/1731394093823/Put/seqid=0 2024-11-12T06:48:15,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742019_1195 (size=13051) 2024-11-12T06:48:15,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742020_1196 (size=31255) 2024-11-12T06:48:15,546 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=438, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/3067afcb0c2b4ea19af799f1519b2d5f 2024-11-12T06:48:15,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/01940bb4521d4ed4b724a92c212ff5f3 is 50, key is test_row_0/B:col10/1731394093823/Put/seqid=0 2024-11-12T06:48:15,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742021_1197 (size=12301) 2024-11-12T06:48:15,557 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/01940bb4521d4ed4b724a92c212ff5f3 2024-11-12T06:48:15,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/024732f08ced4fd78f8d7851cc059350 is 50, key is test_row_0/C:col10/1731394093823/Put/seqid=0 2024-11-12T06:48:15,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742022_1198 (size=12301) 2024-11-12T06:48:15,888 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/d8890594924e486698096810d7634217 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/d8890594924e486698096810d7634217 2024-11-12T06:48:15,892 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/B of 0a0c05661b31342e3af97a578b0b00fc into d8890594924e486698096810d7634217(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:15,892 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:15,892 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/B, priority=13, startTime=1731394095456; duration=0sec 2024-11-12T06:48:15,893 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:15,893 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:B 2024-11-12T06:48:15,948 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/4baca241e6f646eb877f64ef163e9eec as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/4baca241e6f646eb877f64ef163e9eec 2024-11-12T06:48:15,952 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/C of 0a0c05661b31342e3af97a578b0b00fc into 4baca241e6f646eb877f64ef163e9eec(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:15,952 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:15,952 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/C, priority=13, startTime=1731394095456; duration=0sec 2024-11-12T06:48:15,953 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:15,953 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:C 2024-11-12T06:48:15,972 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/024732f08ced4fd78f8d7851cc059350 2024-11-12T06:48:15,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/3067afcb0c2b4ea19af799f1519b2d5f as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/3067afcb0c2b4ea19af799f1519b2d5f 2024-11-12T06:48:15,980 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/3067afcb0c2b4ea19af799f1519b2d5f, entries=150, sequenceid=438, filesize=30.5 K 2024-11-12T06:48:15,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/01940bb4521d4ed4b724a92c212ff5f3 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/01940bb4521d4ed4b724a92c212ff5f3 2024-11-12T06:48:15,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,985 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/01940bb4521d4ed4b724a92c212ff5f3, entries=150, sequenceid=438, filesize=12.0 K 2024-11-12T06:48:15,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/024732f08ced4fd78f8d7851cc059350 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/024732f08ced4fd78f8d7851cc059350 2024-11-12T06:48:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,992 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/024732f08ced4fd78f8d7851cc059350, entries=150, sequenceid=438, filesize=12.0 K 2024-11-12T06:48:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,992 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=0 B/0 for 0a0c05661b31342e3af97a578b0b00fc in 493ms, sequenceid=438, compaction requested=false 2024-11-12T06:48:15,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:15,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-12T06:48:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-12T06:48:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-12T06:48:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0240 sec 2024-11-12T06:48:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,999 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.0290 sec 2024-11-12T06:48:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:16,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T06:48:16,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:16,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:16,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:16,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:16,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:16,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:16,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111201d6490b82ce4141b11aeb46ed6cc2dc_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394096042/Put/seqid=0 2024-11-12T06:48:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742023_1199 (size=22618) 2024-11-12T06:48:16,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,058 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:16,064 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111201d6490b82ce4141b11aeb46ed6cc2dc_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111201d6490b82ce4141b11aeb46ed6cc2dc_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:16,066 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/1cb9a636f1c3422d861539828ccc2dde, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:16,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/1cb9a636f1c3422d861539828ccc2dde is 175, key is test_row_0/A:col10/1731394096042/Put/seqid=0 2024-11-12T06:48:16,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742024_1200 (size=66023) 2024-11-12T06:48:16,073 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=451, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/1cb9a636f1c3422d861539828ccc2dde 2024-11-12T06:48:16,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731394096066, max timestamp=1731394096077Num uniques 859; 2024-11-12T06:48:16,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-12T06:48:16,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731394096066, max timestamp=1731394096077Num uniques 859; 2024-11-12T06:48:16,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-12T06:48:16,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731394096066, max timestamp=1731394096077Num uniques 859; 2024-11-12T06:48:16,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/4631dec5f73f4026a3dacc7534197d12 is 50, key is test_row_0/B:col10/1731394096042/Put/seqid=0 2024-11-12T06:48:16,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-12T06:48:16,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742025_1201 (size=12301) 2024-11-12T06:48:16,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:16,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:16,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394156101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:16,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394156102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:16,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:16,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:16,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394156102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:16,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394156103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:16,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:16,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:16,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394156105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:16,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:16,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:16,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:16,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:16,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T06:48:16,118 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:16,118 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:48:16,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-12T06:48:16,120 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:48:16,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T06:48:16,122 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:48:16,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:48:16,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394156217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:16,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394156217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:16,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:16,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:16,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394156217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:16,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:16,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394156217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:16,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:16,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:16,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394156217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:16,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T06:48:16,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T06:48:16,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:16,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:16,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:16,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:16,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:16,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:16,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T06:48:16,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:16,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:16,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:16,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:16,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:16,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394156427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:16,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:16,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394156427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394156427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394156427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:16,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394156428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:16,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:16,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:16,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T06:48:16,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/4631dec5f73f4026a3dacc7534197d12 2024-11-12T06:48:16,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/722bb14a9fbd435faa8564d145f61244 is 50, key is test_row_0/C:col10/1731394096042/Put/seqid=0 2024-11-12T06:48:16,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742026_1202 (size=12301) 2024-11-12T06:48:16,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T06:48:16,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:16,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:16,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:16,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:16,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:16,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T06:48:16,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:16,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:16,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:16,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:16,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:16,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:16,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394156737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:16,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394156737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:16,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394156737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:16,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394156737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:16,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T06:48:16,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394156748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:16,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T06:48:16,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:16,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:16,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:16,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:16,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:16,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/722bb14a9fbd435faa8564d145f61244 2024-11-12T06:48:16,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/1cb9a636f1c3422d861539828ccc2dde as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/1cb9a636f1c3422d861539828ccc2dde 2024-11-12T06:48:16,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/1cb9a636f1c3422d861539828ccc2dde, entries=350, sequenceid=451, filesize=64.5 K 2024-11-12T06:48:16,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/4631dec5f73f4026a3dacc7534197d12 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/4631dec5f73f4026a3dacc7534197d12 2024-11-12T06:48:16,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/4631dec5f73f4026a3dacc7534197d12, entries=150, sequenceid=451, filesize=12.0 K 2024-11-12T06:48:16,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/722bb14a9fbd435faa8564d145f61244 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/722bb14a9fbd435faa8564d145f61244 2024-11-12T06:48:16,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/722bb14a9fbd435faa8564d145f61244, entries=150, sequenceid=451, filesize=12.0 K 2024-11-12T06:48:16,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 0a0c05661b31342e3af97a578b0b00fc in 882ms, sequenceid=451, compaction requested=true 2024-11-12T06:48:16,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:16,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:48:16,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:16,925 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:16,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:48:16,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:16,925 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:16,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:48:16,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:16,926 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:16,926 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:16,926 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/B is initiating minor compaction (all files) 2024-11-12T06:48:16,926 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/A is initiating minor compaction (all files) 2024-11-12T06:48:16,926 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/B in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:16,926 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/A in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:16,926 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/43a75e5252a94e8490bede971f744f41, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/3067afcb0c2b4ea19af799f1519b2d5f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/1cb9a636f1c3422d861539828ccc2dde] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=126.3 K 2024-11-12T06:48:16,926 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/d8890594924e486698096810d7634217, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/01940bb4521d4ed4b724a92c212ff5f3, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/4631dec5f73f4026a3dacc7534197d12] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=36.8 K 2024-11-12T06:48:16,926 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:16,926 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. files: [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/43a75e5252a94e8490bede971f744f41, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/3067afcb0c2b4ea19af799f1519b2d5f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/1cb9a636f1c3422d861539828ccc2dde] 2024-11-12T06:48:16,927 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting d8890594924e486698096810d7634217, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731394093726 2024-11-12T06:48:16,927 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43a75e5252a94e8490bede971f744f41, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731394093726 2024-11-12T06:48:16,927 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 01940bb4521d4ed4b724a92c212ff5f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731394093823 2024-11-12T06:48:16,927 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3067afcb0c2b4ea19af799f1519b2d5f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731394093823 2024-11-12T06:48:16,927 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 4631dec5f73f4026a3dacc7534197d12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731394096031 2024-11-12T06:48:16,928 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cb9a636f1c3422d861539828ccc2dde, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731394096026 2024-11-12T06:48:16,933 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:16,936 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#B#compaction#163 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:16,936 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111208a323b199734e318f1af2a93ef339ef_0a0c05661b31342e3af97a578b0b00fc store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:16,936 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/8f83067d4a3f45199f2c6e0708b7b8f7 is 50, key is test_row_0/B:col10/1731394096042/Put/seqid=0 2024-11-12T06:48:16,939 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111208a323b199734e318f1af2a93ef339ef_0a0c05661b31342e3af97a578b0b00fc, store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:16,939 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111208a323b199734e318f1af2a93ef339ef_0a0c05661b31342e3af97a578b0b00fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:16,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742027_1203 (size=13153) 2024-11-12T06:48:16,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742028_1204 (size=4469) 2024-11-12T06:48:17,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T06:48:17,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:17,039 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-12T06:48:17,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:17,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:17,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:17,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:17,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:17,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:17,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120b43a311eed44d18b0df1498836d7f82_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394096104/Put/seqid=0 2024-11-12T06:48:17,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742029_1205 (size=12454) 2024-11-12T06:48:17,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:17,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T06:48:17,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394157256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:17,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394157257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:17,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394157257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394157257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:17,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394157259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:17,348 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/8f83067d4a3f45199f2c6e0708b7b8f7 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/8f83067d4a3f45199f2c6e0708b7b8f7 2024-11-12T06:48:17,353 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/B of 0a0c05661b31342e3af97a578b0b00fc into 8f83067d4a3f45199f2c6e0708b7b8f7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:17,353 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:17,353 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/B, priority=13, startTime=1731394096925; duration=0sec 2024-11-12T06:48:17,353 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#A#compaction#162 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:17,353 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:17,353 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:B 2024-11-12T06:48:17,353 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:17,353 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/1edc518c72f94c4fac41a6913f7019a0 is 175, key is test_row_0/A:col10/1731394096042/Put/seqid=0 2024-11-12T06:48:17,354 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:17,354 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/C is initiating minor compaction (all files) 2024-11-12T06:48:17,354 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/C in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:17,354 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/4baca241e6f646eb877f64ef163e9eec, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/024732f08ced4fd78f8d7851cc059350, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/722bb14a9fbd435faa8564d145f61244] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=36.8 K 2024-11-12T06:48:17,355 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 4baca241e6f646eb877f64ef163e9eec, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731394093726 2024-11-12T06:48:17,355 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 024732f08ced4fd78f8d7851cc059350, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731394093823 2024-11-12T06:48:17,355 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 722bb14a9fbd435faa8564d145f61244, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731394096031 2024-11-12T06:48:17,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742030_1206 (size=32107) 2024-11-12T06:48:17,363 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#C#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:17,364 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/ed4e592125314ce1ba639ab9b8c65458 is 50, key is test_row_0/C:col10/1731394096042/Put/seqid=0 2024-11-12T06:48:17,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:17,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742031_1207 (size=13153) 2024-11-12T06:48:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394157367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:17,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394157367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:17,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394157367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:17,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:17,453 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120b43a311eed44d18b0df1498836d7f82_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120b43a311eed44d18b0df1498836d7f82_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:17,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/84f9eb1b901c402f912d9a9be92bb237, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:17,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/84f9eb1b901c402f912d9a9be92bb237 is 175, key is test_row_0/A:col10/1731394096104/Put/seqid=0 2024-11-12T06:48:17,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742032_1208 (size=31255) 2024-11-12T06:48:17,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:17,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394157577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:17,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:17,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:17,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394157577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:17,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:17,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:17,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394157577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:17,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:17,762 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/1edc518c72f94c4fac41a6913f7019a0 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/1edc518c72f94c4fac41a6913f7019a0 2024-11-12T06:48:17,767 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/A of 0a0c05661b31342e3af97a578b0b00fc into 1edc518c72f94c4fac41a6913f7019a0(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:17,767 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:17,768 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/A, priority=13, startTime=1731394096925; duration=0sec 2024-11-12T06:48:17,768 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:17,768 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:A 2024-11-12T06:48:17,772 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/ed4e592125314ce1ba639ab9b8c65458 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/ed4e592125314ce1ba639ab9b8c65458 2024-11-12T06:48:17,776 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/C of 0a0c05661b31342e3af97a578b0b00fc into ed4e592125314ce1ba639ab9b8c65458(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:17,776 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:17,776 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/C, priority=13, startTime=1731394096925; duration=0sec 2024-11-12T06:48:17,777 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:17,777 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:C 2024-11-12T06:48:17,860 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=485, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/84f9eb1b901c402f912d9a9be92bb237 2024-11-12T06:48:17,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/75b72315a14d44d9bd51bea53f83cf55 is 50, key is test_row_0/B:col10/1731394096104/Put/seqid=0 2024-11-12T06:48:17,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742033_1209 (size=12301) 2024-11-12T06:48:17,871 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/75b72315a14d44d9bd51bea53f83cf55 2024-11-12T06:48:17,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/bf89fb36bd7e4496bdcb20ff35746353 is 50, key is test_row_0/C:col10/1731394096104/Put/seqid=0 2024-11-12T06:48:17,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742034_1210 (size=12301) 2024-11-12T06:48:17,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:17,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394157897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:17,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:17,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394157897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:17,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:17,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:17,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394157897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:17,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:17,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T06:48:18,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394158267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:18,269 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_1' in 0a0c05661b31342e3af97a578b0b00fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:48:18,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:18,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:18,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:18,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394158277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:18,280 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_1' in 0a0c05661b31342e3af97a578b0b00fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:48:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:18,282 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/bf89fb36bd7e4496bdcb20ff35746353 2024-11-12T06:48:18,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/84f9eb1b901c402f912d9a9be92bb237 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/84f9eb1b901c402f912d9a9be92bb237 2024-11-12T06:48:18,291 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/84f9eb1b901c402f912d9a9be92bb237, entries=150, sequenceid=485, filesize=30.5 K 2024-11-12T06:48:18,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/75b72315a14d44d9bd51bea53f83cf55 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/75b72315a14d44d9bd51bea53f83cf55 2024-11-12T06:48:18,296 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/75b72315a14d44d9bd51bea53f83cf55, entries=150, sequenceid=485, filesize=12.0 K 2024-11-12T06:48:18,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/bf89fb36bd7e4496bdcb20ff35746353 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/bf89fb36bd7e4496bdcb20ff35746353 2024-11-12T06:48:18,301 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/bf89fb36bd7e4496bdcb20ff35746353, entries=150, sequenceid=485, filesize=12.0 K 2024-11-12T06:48:18,302 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for 0a0c05661b31342e3af97a578b0b00fc in 1263ms, sequenceid=485, compaction requested=false 2024-11-12T06:48:18,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:18,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:18,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-12T06:48:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-12T06:48:18,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-12T06:48:18,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1820 sec 2024-11-12T06:48:18,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.1870 sec 2024-11-12T06:48:18,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:18,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T06:48:18,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:18,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:18,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:18,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:18,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:18,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:18,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e45028d253954fdb8eda4e000e57d21f_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394098409/Put/seqid=0 2024-11-12T06:48:18,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742035_1211 (size=14994) 2024-11-12T06:48:18,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:18,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394158448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:18,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394158449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:18,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:18,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:18,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:18,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394158450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:18,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:18,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:18,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:18,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394158557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:18,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:18,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:18,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:18,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394158566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:18,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394158567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:18,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394158767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:18,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394158777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:18,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394158778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:18,840 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:18,851 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e45028d253954fdb8eda4e000e57d21f_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e45028d253954fdb8eda4e000e57d21f_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:18,852 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/9f04ed4bed1a4dd5a8ac0ecf320f3415, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:18,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/9f04ed4bed1a4dd5a8ac0ecf320f3415 is 175, key is test_row_0/A:col10/1731394098409/Put/seqid=0 2024-11-12T06:48:18,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742036_1212 (size=39949) 2024-11-12T06:48:19,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394159077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:19,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394159087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:19,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394159097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:19,269 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=501, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/9f04ed4bed1a4dd5a8ac0ecf320f3415 2024-11-12T06:48:19,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/1d7a9e204902453496753e7c3d31bdda is 50, key is test_row_0/B:col10/1731394098409/Put/seqid=0 2024-11-12T06:48:19,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742037_1213 (size=12301) 2024-11-12T06:48:19,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:19,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394159587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:19,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394159597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:19,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:19,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394159607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:19,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/1d7a9e204902453496753e7c3d31bdda 2024-11-12T06:48:19,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/e0962213813446219dad3e8fe0fee2a3 is 50, key is test_row_0/C:col10/1731394098409/Put/seqid=0 2024-11-12T06:48:19,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742038_1214 (size=12301) 2024-11-12T06:48:20,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/e0962213813446219dad3e8fe0fee2a3 2024-11-12T06:48:20,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/9f04ed4bed1a4dd5a8ac0ecf320f3415 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/9f04ed4bed1a4dd5a8ac0ecf320f3415 2024-11-12T06:48:20,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/9f04ed4bed1a4dd5a8ac0ecf320f3415, entries=200, sequenceid=501, filesize=39.0 K 2024-11-12T06:48:20,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/1d7a9e204902453496753e7c3d31bdda as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/1d7a9e204902453496753e7c3d31bdda 2024-11-12T06:48:20,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/1d7a9e204902453496753e7c3d31bdda, entries=150, sequenceid=501, filesize=12.0 K 2024-11-12T06:48:20,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/e0962213813446219dad3e8fe0fee2a3 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/e0962213813446219dad3e8fe0fee2a3 2024-11-12T06:48:20,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/e0962213813446219dad3e8fe0fee2a3, entries=150, sequenceid=501, filesize=12.0 K 2024-11-12T06:48:20,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 0a0c05661b31342e3af97a578b0b00fc in 1700ms, sequenceid=501, compaction requested=true 2024-11-12T06:48:20,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:20,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:48:20,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:20,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:48:20,111 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:20,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:20,111 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:20,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a0c05661b31342e3af97a578b0b00fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:48:20,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:20,113 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:20,113 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:20,113 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/A is initiating minor compaction (all files) 2024-11-12T06:48:20,113 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/B is initiating minor compaction (all files) 2024-11-12T06:48:20,114 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/B in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:20,114 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/A in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:20,114 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/1edc518c72f94c4fac41a6913f7019a0, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/84f9eb1b901c402f912d9a9be92bb237, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/9f04ed4bed1a4dd5a8ac0ecf320f3415] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=100.9 K 2024-11-12T06:48:20,114 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/8f83067d4a3f45199f2c6e0708b7b8f7, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/75b72315a14d44d9bd51bea53f83cf55, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/1d7a9e204902453496753e7c3d31bdda] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=36.9 K 2024-11-12T06:48:20,114 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:20,114 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. files: [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/1edc518c72f94c4fac41a6913f7019a0, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/84f9eb1b901c402f912d9a9be92bb237, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/9f04ed4bed1a4dd5a8ac0ecf320f3415] 2024-11-12T06:48:20,114 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f83067d4a3f45199f2c6e0708b7b8f7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731394096031 2024-11-12T06:48:20,114 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1edc518c72f94c4fac41a6913f7019a0, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731394096031 2024-11-12T06:48:20,114 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 75b72315a14d44d9bd51bea53f83cf55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731394096100 2024-11-12T06:48:20,114 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84f9eb1b901c402f912d9a9be92bb237, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731394096100 2024-11-12T06:48:20,115 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d7a9e204902453496753e7c3d31bdda, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731394097256 2024-11-12T06:48:20,115 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f04ed4bed1a4dd5a8ac0ecf320f3415, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731394097256 2024-11-12T06:48:20,120 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:20,122 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111287e7ae605c0648349b295cef2dabe6da_0a0c05661b31342e3af97a578b0b00fc store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:20,122 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#B#compaction#172 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:20,123 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/25617f60fb6e41e9821f2961a275d8c5 is 50, key is test_row_0/B:col10/1731394098409/Put/seqid=0 2024-11-12T06:48:20,123 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111287e7ae605c0648349b295cef2dabe6da_0a0c05661b31342e3af97a578b0b00fc, store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:20,123 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111287e7ae605c0648349b295cef2dabe6da_0a0c05661b31342e3af97a578b0b00fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:20,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742039_1215 (size=13255) 2024-11-12T06:48:20,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742040_1216 (size=4469) 2024-11-12T06:48:20,129 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#A#compaction#171 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:20,130 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/a77f5c611def4235b3cdb6f32e545a4a is 175, key is test_row_0/A:col10/1731394098409/Put/seqid=0 2024-11-12T06:48:20,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742041_1217 (size=32209) 2024-11-12T06:48:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T06:48:20,287 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:20,287 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:48:20,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-12T06:48:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T06:48:20,289 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:48:20,289 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:48:20,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:48:20,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:20,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T06:48:20,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:20,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:20,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:20,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:20,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:20,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:20,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c6a9430ebb7c43d88c2e69dbb4b8b32a_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394098449/Put/seqid=0 2024-11-12T06:48:20,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742042_1218 (size=14994) 2024-11-12T06:48:20,320 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:20,324 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c6a9430ebb7c43d88c2e69dbb4b8b32a_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c6a9430ebb7c43d88c2e69dbb4b8b32a_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:20,325 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/a218dd3ce6fa49819718901370d7b8df, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:20,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/a218dd3ce6fa49819718901370d7b8df is 175, key is test_row_0/A:col10/1731394098449/Put/seqid=0 2024-11-12T06:48:20,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742043_1219 (size=39949) 2024-11-12T06:48:20,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394160327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:20,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:20,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:20,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:20,334 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=521, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/a218dd3ce6fa49819718901370d7b8df 2024-11-12T06:48:20,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394160328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:20,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/022803098c1a422f92f62b34c2e20db9 is 50, key is test_row_0/B:col10/1731394098449/Put/seqid=0 2024-11-12T06:48:20,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742044_1220 (size=12301) 2024-11-12T06:48:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T06:48:20,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394160437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:20,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:20,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:20,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:20,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-12T06:48:20,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:20,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:20,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:20,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:20,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:20,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394160446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:20,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:20,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:20,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:20,530 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/25617f60fb6e41e9821f2961a275d8c5 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/25617f60fb6e41e9821f2961a275d8c5 2024-11-12T06:48:20,535 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/B of 0a0c05661b31342e3af97a578b0b00fc into 25617f60fb6e41e9821f2961a275d8c5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:20,535 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:20,535 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/B, priority=13, startTime=1731394100111; duration=0sec 2024-11-12T06:48:20,535 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:20,536 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:B 2024-11-12T06:48:20,536 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:20,537 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:20,537 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): 0a0c05661b31342e3af97a578b0b00fc/C is initiating minor compaction (all files) 2024-11-12T06:48:20,537 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a0c05661b31342e3af97a578b0b00fc/C in TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:20,537 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/ed4e592125314ce1ba639ab9b8c65458, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/bf89fb36bd7e4496bdcb20ff35746353, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/e0962213813446219dad3e8fe0fee2a3] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp, totalSize=36.9 K 2024-11-12T06:48:20,537 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting ed4e592125314ce1ba639ab9b8c65458, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731394096031 2024-11-12T06:48:20,538 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting bf89fb36bd7e4496bdcb20ff35746353, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731394096100 2024-11-12T06:48:20,538 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting e0962213813446219dad3e8fe0fee2a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731394097256 2024-11-12T06:48:20,541 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/a77f5c611def4235b3cdb6f32e545a4a as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/a77f5c611def4235b3cdb6f32e545a4a 2024-11-12T06:48:20,546 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a0c05661b31342e3af97a578b0b00fc#C#compaction#175 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:20,546 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/537b716ccb464b6f950ab459cd49199a is 50, key is test_row_0/C:col10/1731394098409/Put/seqid=0 2024-11-12T06:48:20,549 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/A of 0a0c05661b31342e3af97a578b0b00fc into a77f5c611def4235b3cdb6f32e545a4a(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:20,549 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:20,549 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/A, priority=13, startTime=1731394100111; duration=0sec 2024-11-12T06:48:20,549 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:20,549 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:A 2024-11-12T06:48:20,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742045_1221 (size=13255) 2024-11-12T06:48:20,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-12T06:48:20,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:20,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:20,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:20,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:20,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:20,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:20,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T06:48:20,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59640 deadline: 1731394160607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:20,609 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_2' in 0a0c05661b31342e3af97a578b0b00fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:48:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:20,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:20,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:20,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59686 deadline: 1731394160617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59708 deadline: 1731394160617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:20,623 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_0' in 0a0c05661b31342e3af97a578b0b00fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:48:20,623 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_0' in 0a0c05661b31342e3af97a578b0b00fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:48:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:20,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:20,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394160647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:20,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:20,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:20,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:20,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:20,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394160657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:20,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/022803098c1a422f92f62b34c2e20db9 2024-11-12T06:48:20,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-12T06:48:20,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:20,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:20,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:20,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:20,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:20,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:20,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/5b96bfa5a4424a1f8565ccb604f0a587 is 50, key is test_row_0/C:col10/1731394098449/Put/seqid=0 2024-11-12T06:48:20,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742046_1222 (size=12301) 2024-11-12T06:48:20,888 INFO [Thread-551 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-551. 2024-11-12T06:48:20,888 DEBUG [Thread-551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:48:20,888 DEBUG [Thread-551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:20,888 DEBUG [Thread-551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:20,888 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:48:20,889 INFO [Thread-553 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-553. 2024-11-12T06:48:20,889 DEBUG [Thread-553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:48:20,889 DEBUG [Thread-553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:20,889 DEBUG [Thread-553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:20,889 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:48:20,892 INFO [Thread-549 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-549. 2024-11-12T06:48:20,892 DEBUG [Thread-549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:48:20,892 DEBUG [Thread-549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:20,892 DEBUG [Thread-549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:20,892 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:48:20,899 INFO [Thread-555 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-555. 2024-11-12T06:48:20,899 DEBUG [Thread-555 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:48:20,899 DEBUG [Thread-555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:20,899 DEBUG [Thread-555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:20,899 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:48:20,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-12T06:48:20,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:20,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:20,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:20,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:20,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T06:48:20,958 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/537b716ccb464b6f950ab459cd49199a as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/537b716ccb464b6f950ab459cd49199a 2024-11-12T06:48:20,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59698 deadline: 1731394160957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:20,964 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a0c05661b31342e3af97a578b0b00fc/C of 0a0c05661b31342e3af97a578b0b00fc into 537b716ccb464b6f950ab459cd49199a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:20,964 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:20,965 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., storeName=0a0c05661b31342e3af97a578b0b00fc/C, priority=13, startTime=1731394100111; duration=0sec 2024-11-12T06:48:20,965 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:20,965 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a0c05661b31342e3af97a578b0b00fc:C 2024-11-12T06:48:20,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59670 deadline: 1731394160967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:20,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:20,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:20,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:21,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-12T06:48:21,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:21,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:21,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:21,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:21,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:21,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:21,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/5b96bfa5a4424a1f8565ccb604f0a587 2024-11-12T06:48:21,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/a218dd3ce6fa49819718901370d7b8df as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/a218dd3ce6fa49819718901370d7b8df 2024-11-12T06:48:21,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/a218dd3ce6fa49819718901370d7b8df, entries=200, sequenceid=521, filesize=39.0 K 2024-11-12T06:48:21,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/022803098c1a422f92f62b34c2e20db9 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/022803098c1a422f92f62b34c2e20db9 2024-11-12T06:48:21,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/022803098c1a422f92f62b34c2e20db9, entries=150, sequenceid=521, filesize=12.0 K 2024-11-12T06:48:21,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/5b96bfa5a4424a1f8565ccb604f0a587 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/5b96bfa5a4424a1f8565ccb604f0a587 2024-11-12T06:48:21,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/5b96bfa5a4424a1f8565ccb604f0a587, entries=150, sequenceid=521, filesize=12.0 K 2024-11-12T06:48:21,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 0a0c05661b31342e3af97a578b0b00fc in 882ms, sequenceid=521, compaction requested=false 2024-11-12T06:48:21,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:21,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-12T06:48:21,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:21,206 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T06:48:21,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:21,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:21,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:21,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:21,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:21,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:21,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d0a6aab572fb4bbb85eada4be321a54f_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394100326/Put/seqid=0 2024-11-12T06:48:21,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742047_1223 (size=12454) 2024-11-12T06:48:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T06:48:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:21,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. as already flushing 2024-11-12T06:48:21,471 INFO [Thread-544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-544. 2024-11-12T06:48:21,472 DEBUG [Thread-544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:48:21,472 DEBUG [Thread-544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:21,472 DEBUG [Thread-544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:21,472 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:48:21,480 INFO [Thread-540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-540. 2024-11-12T06:48:21,480 DEBUG [Thread-540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:48:21,480 DEBUG [Thread-540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:21,480 DEBUG [Thread-540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:21,480 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:48:21,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:21,631 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d0a6aab572fb4bbb85eada4be321a54f_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d0a6aab572fb4bbb85eada4be321a54f_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:21,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/bcf0b22f96074ae38fdb591d9980f7e4, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:21,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/bcf0b22f96074ae38fdb591d9980f7e4 is 175, key is test_row_0/A:col10/1731394100326/Put/seqid=0 2024-11-12T06:48:21,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742048_1224 (size=31255) 2024-11-12T06:48:22,038 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=536, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/bcf0b22f96074ae38fdb591d9980f7e4 2024-11-12T06:48:22,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/347c3bb9023e41e2b2ea7b3766ac6095 is 50, key is test_row_0/B:col10/1731394100326/Put/seqid=0 2024-11-12T06:48:22,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742049_1225 (size=12301) 2024-11-12T06:48:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T06:48:22,501 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/347c3bb9023e41e2b2ea7b3766ac6095 2024-11-12T06:48:22,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/470fb718923c400db901759026c18db5 is 50, key is test_row_0/C:col10/1731394100326/Put/seqid=0 2024-11-12T06:48:22,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742050_1226 (size=12301) 2024-11-12T06:48:22,643 INFO [Thread-538 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-538. 2024-11-12T06:48:22,643 INFO [Thread-546 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-546. 2024-11-12T06:48:22,643 DEBUG [Thread-538 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:48:22,643 DEBUG [Thread-546 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:48:22,643 DEBUG [Thread-538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:22,643 DEBUG [Thread-546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:22,644 DEBUG [Thread-538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:22,644 DEBUG [Thread-546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:22,644 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:48:22,644 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:48:22,648 INFO [Thread-542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-542. 2024-11-12T06:48:22,648 DEBUG [Thread-542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:48:22,649 DEBUG [Thread-542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:22,649 DEBUG [Thread-542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:22,649 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:48:22,920 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/470fb718923c400db901759026c18db5 2024-11-12T06:48:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/bcf0b22f96074ae38fdb591d9980f7e4 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/bcf0b22f96074ae38fdb591d9980f7e4 2024-11-12T06:48:22,930 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/bcf0b22f96074ae38fdb591d9980f7e4, entries=150, sequenceid=536, filesize=30.5 K 2024-11-12T06:48:22,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/347c3bb9023e41e2b2ea7b3766ac6095 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/347c3bb9023e41e2b2ea7b3766ac6095 2024-11-12T06:48:22,936 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/347c3bb9023e41e2b2ea7b3766ac6095, entries=150, sequenceid=536, filesize=12.0 K 2024-11-12T06:48:22,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/470fb718923c400db901759026c18db5 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/470fb718923c400db901759026c18db5 2024-11-12T06:48:22,943 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/470fb718923c400db901759026c18db5, entries=150, sequenceid=536, filesize=12.0 K 2024-11-12T06:48:22,944 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=33.54 KB/34350 for 0a0c05661b31342e3af97a578b0b00fc in 1738ms, sequenceid=536, compaction requested=true 2024-11-12T06:48:22,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 0a0c05661b31342e3af97a578b0b00fc: 2024-11-12T06:48:22,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:22,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-12T06:48:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-12T06:48:22,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-12T06:48:22,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6570 sec 2024-11-12T06:48:22,949 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.6610 sec 2024-11-12T06:48:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T06:48:24,448 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:24,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T06:48:24,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-11-12T06:48:24,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-12T06:48:24,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-12T06:48:24,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-12T06:48:24,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-12T06:48:24,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T06:48:24,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2802 2024-11-12T06:48:24,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2741 2024-11-12T06:48:24,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T06:48:24,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1191 2024-11-12T06:48:24,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3573 rows 2024-11-12T06:48:24,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1185 2024-11-12T06:48:24,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3555 rows 2024-11-12T06:48:24,449 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T06:48:24,449 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T06:48:24,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:24,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:24,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:48:24,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T06:48:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T06:48:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T06:48:24,458 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394104457"}]},"ts":"1731394104457"} 2024-11-12T06:48:24,459 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T06:48:24,459 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T06:48:24,460 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T06:48:24,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0a0c05661b31342e3af97a578b0b00fc, UNASSIGN}] 2024-11-12T06:48:24,462 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0a0c05661b31342e3af97a578b0b00fc, UNASSIGN 2024-11-12T06:48:24,463 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=0a0c05661b31342e3af97a578b0b00fc, regionState=CLOSING, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:24,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0a0c05661b31342e3af97a578b0b00fc, UNASSIGN because future has completed 2024-11-12T06:48:24,464 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T06:48:24,464 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622}] 2024-11-12T06:48:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T06:48:24,616 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:24,617 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T06:48:24,617 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 0a0c05661b31342e3af97a578b0b00fc, disabling compactions & flushes 2024-11-12T06:48:24,617 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:24,617 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:24,617 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. after waiting 0 ms 2024-11-12T06:48:24,617 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:24,617 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing 0a0c05661b31342e3af97a578b0b00fc 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-12T06:48:24,618 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=A 2024-11-12T06:48:24,618 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:24,618 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=B 2024-11-12T06:48:24,618 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:24,618 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0a0c05661b31342e3af97a578b0b00fc, store=C 2024-11-12T06:48:24,618 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:24,627 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112bc8541ea82bc4b6b859d8e3302423e01_0a0c05661b31342e3af97a578b0b00fc is 50, key is test_row_0/A:col10/1731394102647/Put/seqid=0 2024-11-12T06:48:24,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742051_1227 (size=9914) 2024-11-12T06:48:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T06:48:25,033 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:25,042 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112bc8541ea82bc4b6b859d8e3302423e01_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112bc8541ea82bc4b6b859d8e3302423e01_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:25,044 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/26bc026a2a6a4634881f7a9a91e9bf71, store: [table=TestAcidGuarantees family=A region=0a0c05661b31342e3af97a578b0b00fc] 2024-11-12T06:48:25,044 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/26bc026a2a6a4634881f7a9a91e9bf71 is 175, key is test_row_0/A:col10/1731394102647/Put/seqid=0 2024-11-12T06:48:25,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742052_1228 (size=22561) 2024-11-12T06:48:25,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T06:48:25,451 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=544, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/26bc026a2a6a4634881f7a9a91e9bf71 2024-11-12T06:48:25,467 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/4bd9bb0cb7444bae93fd2cc109eaf0d7 is 50, key is test_row_0/B:col10/1731394102647/Put/seqid=0 2024-11-12T06:48:25,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742053_1229 (size=9857) 2024-11-12T06:48:25,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T06:48:25,873 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/4bd9bb0cb7444bae93fd2cc109eaf0d7 2024-11-12T06:48:25,885 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/e1ace98b746c42bdbce019b3e8383264 is 50, key is test_row_0/C:col10/1731394102647/Put/seqid=0 2024-11-12T06:48:25,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742054_1230 (size=9857) 2024-11-12T06:48:25,969 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T06:48:26,292 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/e1ace98b746c42bdbce019b3e8383264 2024-11-12T06:48:26,304 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/A/26bc026a2a6a4634881f7a9a91e9bf71 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/26bc026a2a6a4634881f7a9a91e9bf71 2024-11-12T06:48:26,309 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/26bc026a2a6a4634881f7a9a91e9bf71, entries=100, sequenceid=544, filesize=22.0 K 2024-11-12T06:48:26,310 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/B/4bd9bb0cb7444bae93fd2cc109eaf0d7 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/4bd9bb0cb7444bae93fd2cc109eaf0d7 2024-11-12T06:48:26,315 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/4bd9bb0cb7444bae93fd2cc109eaf0d7, entries=100, sequenceid=544, filesize=9.6 K 2024-11-12T06:48:26,316 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/.tmp/C/e1ace98b746c42bdbce019b3e8383264 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/e1ace98b746c42bdbce019b3e8383264 2024-11-12T06:48:26,319 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/e1ace98b746c42bdbce019b3e8383264, entries=100, sequenceid=544, filesize=9.6 K 2024-11-12T06:48:26,320 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0a0c05661b31342e3af97a578b0b00fc in 1703ms, sequenceid=544, compaction requested=true 2024-11-12T06:48:26,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/09a884bf11824f28839444c806c7d788, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/2d4a0d21c5214e69a0bef834a15e8cc4, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/33640ee0daa348448e7bed1655a0622b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/0e5180caf5134c40b85725d293691dd0, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/c51192ee6be44deb825dc498f80fcc91, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/8632999e62a14116a5e59a841c616e59, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/4b392ea52e464c34adf31bf93b97e663, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/8169947025c84b9eb46bba283c4cef11, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/68021602b6ba43aa8e215cfcbc74ae03, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/3d76a865dd3a423db2252cdad92741c8, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/860331e24d564b50a4ac26400bd2549d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/b1067816533b4346acef019a06dcf7e0, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/44ec13cdb25045f3b038e04d41e1b635, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/196bbfb1d82c459aa35260a07a2b4bd0, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/a78f17ef7122442885195dda8c9f3e3f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/a2ffe4ee50114f37b013a34a7acec6d4, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/7df65ee81c79453b92d913b2c5fefe00, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/9373f530d3674cabaa3c99c5c9683513, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/73a5d86dffe44ef6a8d20bdb1bf63567, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/dad191a7b1b54aaab895d1a1be49192b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/57e02573e8a7464596752d9aa52971ba, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/f6ac5e376de04292925134991d01df4f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/43a75e5252a94e8490bede971f744f41, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/3067afcb0c2b4ea19af799f1519b2d5f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/1cb9a636f1c3422d861539828ccc2dde, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/1edc518c72f94c4fac41a6913f7019a0, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/84f9eb1b901c402f912d9a9be92bb237, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/9f04ed4bed1a4dd5a8ac0ecf320f3415] to archive 2024-11-12T06:48:26,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T06:48:26,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/09a884bf11824f28839444c806c7d788 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/09a884bf11824f28839444c806c7d788 2024-11-12T06:48:26,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/2d4a0d21c5214e69a0bef834a15e8cc4 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/2d4a0d21c5214e69a0bef834a15e8cc4 2024-11-12T06:48:26,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/33640ee0daa348448e7bed1655a0622b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/33640ee0daa348448e7bed1655a0622b 2024-11-12T06:48:26,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/0e5180caf5134c40b85725d293691dd0 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/0e5180caf5134c40b85725d293691dd0 2024-11-12T06:48:26,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/c51192ee6be44deb825dc498f80fcc91 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/c51192ee6be44deb825dc498f80fcc91 2024-11-12T06:48:26,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/8632999e62a14116a5e59a841c616e59 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/8632999e62a14116a5e59a841c616e59 2024-11-12T06:48:26,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/4b392ea52e464c34adf31bf93b97e663 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/4b392ea52e464c34adf31bf93b97e663 2024-11-12T06:48:26,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/8169947025c84b9eb46bba283c4cef11 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/8169947025c84b9eb46bba283c4cef11 2024-11-12T06:48:26,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/68021602b6ba43aa8e215cfcbc74ae03 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/68021602b6ba43aa8e215cfcbc74ae03 2024-11-12T06:48:26,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/3d76a865dd3a423db2252cdad92741c8 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/3d76a865dd3a423db2252cdad92741c8 2024-11-12T06:48:26,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/860331e24d564b50a4ac26400bd2549d to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/860331e24d564b50a4ac26400bd2549d 2024-11-12T06:48:26,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/b1067816533b4346acef019a06dcf7e0 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/b1067816533b4346acef019a06dcf7e0 2024-11-12T06:48:26,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/44ec13cdb25045f3b038e04d41e1b635 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/44ec13cdb25045f3b038e04d41e1b635 2024-11-12T06:48:26,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/196bbfb1d82c459aa35260a07a2b4bd0 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/196bbfb1d82c459aa35260a07a2b4bd0 2024-11-12T06:48:26,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/a78f17ef7122442885195dda8c9f3e3f to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/a78f17ef7122442885195dda8c9f3e3f 2024-11-12T06:48:26,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/a2ffe4ee50114f37b013a34a7acec6d4 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/a2ffe4ee50114f37b013a34a7acec6d4 2024-11-12T06:48:26,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/7df65ee81c79453b92d913b2c5fefe00 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/7df65ee81c79453b92d913b2c5fefe00 2024-11-12T06:48:26,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/9373f530d3674cabaa3c99c5c9683513 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/9373f530d3674cabaa3c99c5c9683513 2024-11-12T06:48:26,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/73a5d86dffe44ef6a8d20bdb1bf63567 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/73a5d86dffe44ef6a8d20bdb1bf63567 2024-11-12T06:48:26,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/dad191a7b1b54aaab895d1a1be49192b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/dad191a7b1b54aaab895d1a1be49192b 2024-11-12T06:48:26,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/57e02573e8a7464596752d9aa52971ba to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/57e02573e8a7464596752d9aa52971ba 2024-11-12T06:48:26,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/f6ac5e376de04292925134991d01df4f to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/f6ac5e376de04292925134991d01df4f 2024-11-12T06:48:26,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/43a75e5252a94e8490bede971f744f41 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/43a75e5252a94e8490bede971f744f41 2024-11-12T06:48:26,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/3067afcb0c2b4ea19af799f1519b2d5f to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/3067afcb0c2b4ea19af799f1519b2d5f 2024-11-12T06:48:26,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/1cb9a636f1c3422d861539828ccc2dde to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/1cb9a636f1c3422d861539828ccc2dde 2024-11-12T06:48:26,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/1edc518c72f94c4fac41a6913f7019a0 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/1edc518c72f94c4fac41a6913f7019a0 2024-11-12T06:48:26,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/84f9eb1b901c402f912d9a9be92bb237 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/84f9eb1b901c402f912d9a9be92bb237 2024-11-12T06:48:26,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/9f04ed4bed1a4dd5a8ac0ecf320f3415 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/9f04ed4bed1a4dd5a8ac0ecf320f3415 2024-11-12T06:48:26,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/0262c680599c4c7db0f6cdafe6027a97, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/1301bdc5312b4067b01236460bf5e338, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/3a73628fd52340e19727a626e2d439f3, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/ecbb9fa8b2da41a69f33cccab7ecb57d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/5af07f9f0ccc43109f970425c5aa0d6f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/6173a94b3ac642b8b6b03e49326b2215, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/7eac3c49bc9b479a9adc5c025d666fa0, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/a5fa4322e4634658a7bb29b09e6279e5, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/a5e0381ef6f341e6aafd5b26b2c4ff15, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/0a6f5d8b908249f2bfc25fc56037b4a3, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/63e7552c7cae4ccbbd344b3e846abf00, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/52166ede5574482a91586d16e0f14c87, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/b668af85ba144f2f85236d2fbf3646e2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/aecb5f98d2544f3d8c4d64b2c82d5286, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/c355430b825e46d4b1216e5fec29fcf4, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/8839b22eec534b22a67c0debd43dac0a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/afba092782554d72ab7dbaeb6942252c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/a56da07cd74c45bbae2aa0ab6c015350, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/f8f682c31e20472990321783ce3ec337, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/23c52894a6e848afb44fc99241738747, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/03f154e9cdcd421693f52deabc2959a2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/d8890594924e486698096810d7634217, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/f4b5a3c225b74277bb4d9fc8c9265088, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/01940bb4521d4ed4b724a92c212ff5f3, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/8f83067d4a3f45199f2c6e0708b7b8f7, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/4631dec5f73f4026a3dacc7534197d12, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/75b72315a14d44d9bd51bea53f83cf55, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/1d7a9e204902453496753e7c3d31bdda] to archive 2024-11-12T06:48:26,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T06:48:26,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/0262c680599c4c7db0f6cdafe6027a97 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/0262c680599c4c7db0f6cdafe6027a97 2024-11-12T06:48:26,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/1301bdc5312b4067b01236460bf5e338 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/1301bdc5312b4067b01236460bf5e338 2024-11-12T06:48:26,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/3a73628fd52340e19727a626e2d439f3 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/3a73628fd52340e19727a626e2d439f3 2024-11-12T06:48:26,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/ecbb9fa8b2da41a69f33cccab7ecb57d to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/ecbb9fa8b2da41a69f33cccab7ecb57d 2024-11-12T06:48:26,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/5af07f9f0ccc43109f970425c5aa0d6f to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/5af07f9f0ccc43109f970425c5aa0d6f 2024-11-12T06:48:26,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/6173a94b3ac642b8b6b03e49326b2215 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/6173a94b3ac642b8b6b03e49326b2215 2024-11-12T06:48:26,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/7eac3c49bc9b479a9adc5c025d666fa0 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/7eac3c49bc9b479a9adc5c025d666fa0 2024-11-12T06:48:26,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/a5fa4322e4634658a7bb29b09e6279e5 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/a5fa4322e4634658a7bb29b09e6279e5 2024-11-12T06:48:26,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/a5e0381ef6f341e6aafd5b26b2c4ff15 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/a5e0381ef6f341e6aafd5b26b2c4ff15 2024-11-12T06:48:26,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/0a6f5d8b908249f2bfc25fc56037b4a3 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/0a6f5d8b908249f2bfc25fc56037b4a3 2024-11-12T06:48:26,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/63e7552c7cae4ccbbd344b3e846abf00 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/63e7552c7cae4ccbbd344b3e846abf00 2024-11-12T06:48:26,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/52166ede5574482a91586d16e0f14c87 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/52166ede5574482a91586d16e0f14c87 2024-11-12T06:48:26,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/b668af85ba144f2f85236d2fbf3646e2 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/b668af85ba144f2f85236d2fbf3646e2 2024-11-12T06:48:26,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/aecb5f98d2544f3d8c4d64b2c82d5286 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/aecb5f98d2544f3d8c4d64b2c82d5286 2024-11-12T06:48:26,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/c355430b825e46d4b1216e5fec29fcf4 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/c355430b825e46d4b1216e5fec29fcf4 2024-11-12T06:48:26,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/8839b22eec534b22a67c0debd43dac0a to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/8839b22eec534b22a67c0debd43dac0a 2024-11-12T06:48:26,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/afba092782554d72ab7dbaeb6942252c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/afba092782554d72ab7dbaeb6942252c 2024-11-12T06:48:26,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/a56da07cd74c45bbae2aa0ab6c015350 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/a56da07cd74c45bbae2aa0ab6c015350 2024-11-12T06:48:26,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/f8f682c31e20472990321783ce3ec337 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/f8f682c31e20472990321783ce3ec337 2024-11-12T06:48:26,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/23c52894a6e848afb44fc99241738747 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/23c52894a6e848afb44fc99241738747 2024-11-12T06:48:26,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/03f154e9cdcd421693f52deabc2959a2 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/03f154e9cdcd421693f52deabc2959a2 2024-11-12T06:48:26,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/d8890594924e486698096810d7634217 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/d8890594924e486698096810d7634217 2024-11-12T06:48:26,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/f4b5a3c225b74277bb4d9fc8c9265088 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/f4b5a3c225b74277bb4d9fc8c9265088 2024-11-12T06:48:26,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/01940bb4521d4ed4b724a92c212ff5f3 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/01940bb4521d4ed4b724a92c212ff5f3 2024-11-12T06:48:26,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/8f83067d4a3f45199f2c6e0708b7b8f7 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/8f83067d4a3f45199f2c6e0708b7b8f7 2024-11-12T06:48:26,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/4631dec5f73f4026a3dacc7534197d12 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/4631dec5f73f4026a3dacc7534197d12 2024-11-12T06:48:26,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/75b72315a14d44d9bd51bea53f83cf55 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/75b72315a14d44d9bd51bea53f83cf55 2024-11-12T06:48:26,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/1d7a9e204902453496753e7c3d31bdda to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/1d7a9e204902453496753e7c3d31bdda 2024-11-12T06:48:26,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/8b41d993bea54a95af3c6bece42ca73a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/81b435dd7b8649c7a58c8ca038b4acdc, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/62d09e23835d466f85beea0af82093d1, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/f766cd8c416c4806ba23243d47fa4f07, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/0beb0b1fa7dd4ab19cd461bf1135d19f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/141b19a23fe3429a99d04ea9bff2c0c5, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/151d8c5d154741afbbaf8a2fa989cfb2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/5717f2cc67544f66b04547215f1ff25b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/43e9be02ca0b4d819e1522e02385bff1, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/35d6a7259d7e4efe907e7d281b1cbde8, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/b37835d6e90c4ac2894f171e8ad09e5d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/2f2a552a5dd0467f911fb9e496fbcb7c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/8905faaf37f44e968b5c79d718abd351, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/008034b457614f28bfca118cd2e9595c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/631f48b40b3949df81205021ab7cf046, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/64ac88ab754a4fc3abee6696c7cb86fa, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/23c4337e156e4b2bafb4f22d9400360a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/379c0c24b02c425b8f2b48ca4093ca5a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/d963aeeb48fb4f238d5b51dd84e8445c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/1fd5c5999e7846d583cf38e2e47a89f2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/397a7eb755154f3ea80d24dfe26a976e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/4baca241e6f646eb877f64ef163e9eec, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/b6d52d4f6fbf484c80e3f927cba11197, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/024732f08ced4fd78f8d7851cc059350, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/ed4e592125314ce1ba639ab9b8c65458, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/722bb14a9fbd435faa8564d145f61244, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/bf89fb36bd7e4496bdcb20ff35746353, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/e0962213813446219dad3e8fe0fee2a3] to archive 2024-11-12T06:48:26,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T06:48:26,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/8b41d993bea54a95af3c6bece42ca73a to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/8b41d993bea54a95af3c6bece42ca73a 2024-11-12T06:48:26,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/81b435dd7b8649c7a58c8ca038b4acdc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/81b435dd7b8649c7a58c8ca038b4acdc 2024-11-12T06:48:26,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/62d09e23835d466f85beea0af82093d1 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/62d09e23835d466f85beea0af82093d1 2024-11-12T06:48:26,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/f766cd8c416c4806ba23243d47fa4f07 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/f766cd8c416c4806ba23243d47fa4f07 2024-11-12T06:48:26,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/0beb0b1fa7dd4ab19cd461bf1135d19f to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/0beb0b1fa7dd4ab19cd461bf1135d19f 2024-11-12T06:48:26,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/141b19a23fe3429a99d04ea9bff2c0c5 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/141b19a23fe3429a99d04ea9bff2c0c5 2024-11-12T06:48:26,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/151d8c5d154741afbbaf8a2fa989cfb2 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/151d8c5d154741afbbaf8a2fa989cfb2 2024-11-12T06:48:26,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/5717f2cc67544f66b04547215f1ff25b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/5717f2cc67544f66b04547215f1ff25b 2024-11-12T06:48:26,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/43e9be02ca0b4d819e1522e02385bff1 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/43e9be02ca0b4d819e1522e02385bff1 2024-11-12T06:48:26,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/35d6a7259d7e4efe907e7d281b1cbde8 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/35d6a7259d7e4efe907e7d281b1cbde8 2024-11-12T06:48:26,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/b37835d6e90c4ac2894f171e8ad09e5d to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/b37835d6e90c4ac2894f171e8ad09e5d 2024-11-12T06:48:26,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/2f2a552a5dd0467f911fb9e496fbcb7c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/2f2a552a5dd0467f911fb9e496fbcb7c 2024-11-12T06:48:26,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/8905faaf37f44e968b5c79d718abd351 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/8905faaf37f44e968b5c79d718abd351 2024-11-12T06:48:26,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/008034b457614f28bfca118cd2e9595c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/008034b457614f28bfca118cd2e9595c 2024-11-12T06:48:26,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/631f48b40b3949df81205021ab7cf046 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/631f48b40b3949df81205021ab7cf046 2024-11-12T06:48:26,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/64ac88ab754a4fc3abee6696c7cb86fa to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/64ac88ab754a4fc3abee6696c7cb86fa 2024-11-12T06:48:26,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/23c4337e156e4b2bafb4f22d9400360a to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/23c4337e156e4b2bafb4f22d9400360a 2024-11-12T06:48:26,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/379c0c24b02c425b8f2b48ca4093ca5a to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/379c0c24b02c425b8f2b48ca4093ca5a 2024-11-12T06:48:26,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/d963aeeb48fb4f238d5b51dd84e8445c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/d963aeeb48fb4f238d5b51dd84e8445c 2024-11-12T06:48:26,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/1fd5c5999e7846d583cf38e2e47a89f2 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/1fd5c5999e7846d583cf38e2e47a89f2 2024-11-12T06:48:26,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/397a7eb755154f3ea80d24dfe26a976e to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/397a7eb755154f3ea80d24dfe26a976e 2024-11-12T06:48:26,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/4baca241e6f646eb877f64ef163e9eec to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/4baca241e6f646eb877f64ef163e9eec 2024-11-12T06:48:26,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/b6d52d4f6fbf484c80e3f927cba11197 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/b6d52d4f6fbf484c80e3f927cba11197 2024-11-12T06:48:26,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/024732f08ced4fd78f8d7851cc059350 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/024732f08ced4fd78f8d7851cc059350 2024-11-12T06:48:26,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/ed4e592125314ce1ba639ab9b8c65458 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/ed4e592125314ce1ba639ab9b8c65458 2024-11-12T06:48:26,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/722bb14a9fbd435faa8564d145f61244 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/722bb14a9fbd435faa8564d145f61244 2024-11-12T06:48:26,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/bf89fb36bd7e4496bdcb20ff35746353 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/bf89fb36bd7e4496bdcb20ff35746353 2024-11-12T06:48:26,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/e0962213813446219dad3e8fe0fee2a3 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/e0962213813446219dad3e8fe0fee2a3 2024-11-12T06:48:26,409 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/recovered.edits/547.seqid, newMaxSeqId=547, maxSeqId=4 2024-11-12T06:48:26,410 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. 2024-11-12T06:48:26,410 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 0a0c05661b31342e3af97a578b0b00fc: Waiting for close lock at 1731394104617Running coprocessor pre-close hooks at 1731394104617Disabling compacts and flushes for region at 1731394104617Disabling writes for close at 1731394104617Obtaining lock to block concurrent updates at 1731394104617Preparing flush snapshotting stores in 0a0c05661b31342e3af97a578b0b00fc at 1731394104617Finished memstore snapshotting TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731394104618 (+1 ms)Flushing stores of TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc. at 1731394104619 (+1 ms)Flushing 0a0c05661b31342e3af97a578b0b00fc/A: creating writer at 1731394104619Flushing mob file 0a0c05661b31342e3af97a578b0b00fc/A: appending metadata at 1731394104627 (+8 ms)Flushing mob file 0a0c05661b31342e3af97a578b0b00fc/A: closing flushed file at 1731394104627Flushing 0a0c05661b31342e3af97a578b0b00fc/A: appending metadata at 1731394105044 (+417 ms)Flushing 0a0c05661b31342e3af97a578b0b00fc/A: closing flushed file at 1731394105044Flushing 0a0c05661b31342e3af97a578b0b00fc/B: creating writer at 1731394105462 (+418 ms)Flushing 0a0c05661b31342e3af97a578b0b00fc/B: appending metadata at 1731394105467 (+5 ms)Flushing 0a0c05661b31342e3af97a578b0b00fc/B: closing flushed file at 1731394105467Flushing 0a0c05661b31342e3af97a578b0b00fc/C: creating writer at 1731394105881 (+414 ms)Flushing 0a0c05661b31342e3af97a578b0b00fc/C: appending metadata at 1731394105884 (+3 ms)Flushing 0a0c05661b31342e3af97a578b0b00fc/C: closing flushed file at 1731394105884Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3acbebe1: reopening flushed file at 1731394106303 (+419 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69f97213: reopening flushed file at 1731394106309 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70945dc1: reopening flushed file at 1731394106315 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0a0c05661b31342e3af97a578b0b00fc in 1703ms, sequenceid=544, compaction requested=true at 1731394106320 (+5 ms)Writing region close event to WAL at 1731394106406 (+86 ms)Running coprocessor post-close hooks at 1731394106410 (+4 ms)Closed at 1731394106410 2024-11-12T06:48:26,412 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,412 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=0a0c05661b31342e3af97a578b0b00fc, regionState=CLOSED 2024-11-12T06:48:26,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 because future has completed 2024-11-12T06:48:26,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-12T06:48:26,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 0a0c05661b31342e3af97a578b0b00fc, server=9f0beee3d6b6,38777,1731394047622 in 1.9500 sec 2024-11-12T06:48:26,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-12T06:48:26,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0a0c05661b31342e3af97a578b0b00fc, UNASSIGN in 1.9550 sec 2024-11-12T06:48:26,419 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-12T06:48:26,419 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9570 sec 2024-11-12T06:48:26,420 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394106420"}]},"ts":"1731394106420"} 2024-11-12T06:48:26,421 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T06:48:26,421 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T06:48:26,423 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9690 sec 2024-11-12T06:48:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T06:48:26,608 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:26,610 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T06:48:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:48:26,614 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:48:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T06:48:26,616 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:48:26,622 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,626 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A, FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B, FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C, FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/recovered.edits] 2024-11-12T06:48:26,630 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/26bc026a2a6a4634881f7a9a91e9bf71 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/26bc026a2a6a4634881f7a9a91e9bf71 2024-11-12T06:48:26,632 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/a218dd3ce6fa49819718901370d7b8df to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/a218dd3ce6fa49819718901370d7b8df 2024-11-12T06:48:26,634 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/a77f5c611def4235b3cdb6f32e545a4a to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/a77f5c611def4235b3cdb6f32e545a4a 2024-11-12T06:48:26,636 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/bcf0b22f96074ae38fdb591d9980f7e4 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/A/bcf0b22f96074ae38fdb591d9980f7e4 2024-11-12T06:48:26,640 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/022803098c1a422f92f62b34c2e20db9 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/022803098c1a422f92f62b34c2e20db9 2024-11-12T06:48:26,641 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/25617f60fb6e41e9821f2961a275d8c5 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/25617f60fb6e41e9821f2961a275d8c5 2024-11-12T06:48:26,644 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/347c3bb9023e41e2b2ea7b3766ac6095 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/347c3bb9023e41e2b2ea7b3766ac6095 2024-11-12T06:48:26,646 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/4bd9bb0cb7444bae93fd2cc109eaf0d7 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/B/4bd9bb0cb7444bae93fd2cc109eaf0d7 2024-11-12T06:48:26,649 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/470fb718923c400db901759026c18db5 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/470fb718923c400db901759026c18db5 2024-11-12T06:48:26,651 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/537b716ccb464b6f950ab459cd49199a to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/537b716ccb464b6f950ab459cd49199a 2024-11-12T06:48:26,652 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/5b96bfa5a4424a1f8565ccb604f0a587 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/5b96bfa5a4424a1f8565ccb604f0a587 2024-11-12T06:48:26,654 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/e1ace98b746c42bdbce019b3e8383264 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/C/e1ace98b746c42bdbce019b3e8383264 2024-11-12T06:48:26,657 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/recovered.edits/547.seqid to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc/recovered.edits/547.seqid 2024-11-12T06:48:26,658 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,658 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T06:48:26,659 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T06:48:26,660 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-12T06:48:26,667 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111201d6490b82ce4141b11aeb46ed6cc2dc_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111201d6490b82ce4141b11aeb46ed6cc2dc_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,669 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120b43a311eed44d18b0df1498836d7f82_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120b43a311eed44d18b0df1498836d7f82_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,672 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120d73253385144519b63a92e7b0a81dad_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120d73253385144519b63a92e7b0a81dad_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,674 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112181cb2533b1542949d9302bf523618ab_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112181cb2533b1542949d9302bf523618ab_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,676 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111238a140d5f32c41cb9c1ced4b3feb9737_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111238a140d5f32c41cb9c1ced4b3feb9737_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,678 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123cc750a8b54843f99e2be1af6e99928d_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123cc750a8b54843f99e2be1af6e99928d_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,680 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124316cd56fe4a472a913c2345bbfbc700_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124316cd56fe4a472a913c2345bbfbc700_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,682 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124652878bbdec49efacdcc16c1040ffac_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124652878bbdec49efacdcc16c1040ffac_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,685 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124b5171c6bead4860aff48ed518cd8327_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124b5171c6bead4860aff48ed518cd8327_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,687 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125d4c1841eb524d5d8616eac19efebc75_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125d4c1841eb524d5d8616eac19efebc75_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,690 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111264b88e42c351455eb294b35d72378bd7_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111264b88e42c351455eb294b35d72378bd7_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,691 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128f69ef26d03e41c7a107c67f0ad465b1_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128f69ef26d03e41c7a107c67f0ad465b1_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,692 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129e812c611d8c452083ec998b65e4a6a7_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129e812c611d8c452083ec998b65e4a6a7_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,693 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a34ab5d6fd1e43f39510ba722c8d3d11_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a34ab5d6fd1e43f39510ba722c8d3d11_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,695 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a568eedc6c464b83903c8a78044ee821_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a568eedc6c464b83903c8a78044ee821_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,696 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112bac0114268b244b5b68b70e3bfa3e19d_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112bac0114268b244b5b68b70e3bfa3e19d_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,697 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112bc8541ea82bc4b6b859d8e3302423e01_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112bc8541ea82bc4b6b859d8e3302423e01_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,698 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c085453e8d6841a3b3409d6703a1f3d9_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c085453e8d6841a3b3409d6703a1f3d9_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,700 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c6a9430ebb7c43d88c2e69dbb4b8b32a_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c6a9430ebb7c43d88c2e69dbb4b8b32a_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,701 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d0a6aab572fb4bbb85eada4be321a54f_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d0a6aab572fb4bbb85eada4be321a54f_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,702 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e45028d253954fdb8eda4e000e57d21f_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e45028d253954fdb8eda4e000e57d21f_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,704 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e481007355ca4e7d831c829c63ef3de2_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e481007355ca4e7d831c829c63ef3de2_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,705 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e86a7b99f5a04991b7535521cc42ce0a_0a0c05661b31342e3af97a578b0b00fc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e86a7b99f5a04991b7535521cc42ce0a_0a0c05661b31342e3af97a578b0b00fc 2024-11-12T06:48:26,705 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T06:48:26,708 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:48:26,711 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T06:48:26,713 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T06:48:26,714 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:48:26,715 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T06:48:26,715 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731394106715"}]},"ts":"9223372036854775807"} 2024-11-12T06:48:26,718 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T06:48:26,718 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0a0c05661b31342e3af97a578b0b00fc, NAME => 'TestAcidGuarantees,,1731394077701.0a0c05661b31342e3af97a578b0b00fc.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T06:48:26,718 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T06:48:26,718 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731394106718"}]},"ts":"9223372036854775807"} 2024-11-12T06:48:26,721 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T06:48:26,722 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:48:26,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 112 msec 2024-11-12T06:48:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T06:48:26,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T06:48:26,728 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:26,735 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_455841092_22 at /127.0.0.1:35358 [Waiting for operation #670] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_455841092_22 at /127.0.0.1:35320 [Waiting for operation #749] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/cluster_a8cb0a6a-8309-0165-eb73-378e9ec04732/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-553601269_22 at /127.0.0.1:45420 [Waiting for operation #1089] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/cluster_a8cb0a6a-8309-0165-eb73-378e9ec04732/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-553601269_22 at /127.0.0.1:39144 [Waiting for operation #1104] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=313 (was 273) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6753 (was 6914) 2024-11-12T06:48:26,743 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=313, ProcessCount=11, AvailableMemoryMB=6752 2024-11-12T06:48:26,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T06:48:26,744 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T06:48:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T06:48:26,746 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T06:48:26,746 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:26,747 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-12T06:48:26,747 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T06:48:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-12T06:48:26,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742055_1231 (size=1061) 2024-11-12T06:48:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-12T06:48:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-12T06:48:27,156 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e 2024-11-12T06:48:27,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742056_1232 (size=53) 2024-11-12T06:48:27,229 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T06:48:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-12T06:48:27,565 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T06:48:27,565 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing abf9890bb5fa9b12fd7d1c33ce62b875, disabling compactions & flushes 2024-11-12T06:48:27,565 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:27,565 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:27,565 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. after waiting 0 ms 2024-11-12T06:48:27,565 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:27,565 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:27,565 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for abf9890bb5fa9b12fd7d1c33ce62b875: Waiting for close lock at 1731394107565Disabling compacts and flushes for region at 1731394107565Disabling writes for close at 1731394107565Writing region close event to WAL at 1731394107565Closed at 1731394107565 2024-11-12T06:48:27,567 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T06:48:27,567 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731394107567"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731394107567"}]},"ts":"1731394107567"} 2024-11-12T06:48:27,570 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T06:48:27,571 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T06:48:27,572 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394107571"}]},"ts":"1731394107571"} 2024-11-12T06:48:27,574 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T06:48:27,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=abf9890bb5fa9b12fd7d1c33ce62b875, ASSIGN}] 2024-11-12T06:48:27,577 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=abf9890bb5fa9b12fd7d1c33ce62b875, ASSIGN 2024-11-12T06:48:27,578 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=abf9890bb5fa9b12fd7d1c33ce62b875, ASSIGN; state=OFFLINE, location=9f0beee3d6b6,38777,1731394047622; forceNewPlan=false, retain=false 2024-11-12T06:48:27,728 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=abf9890bb5fa9b12fd7d1c33ce62b875, regionState=OPENING, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:27,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=abf9890bb5fa9b12fd7d1c33ce62b875, ASSIGN because future has completed 2024-11-12T06:48:27,730 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622}] 2024-11-12T06:48:27,885 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:27,885 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} 2024-11-12T06:48:27,885 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:27,885 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T06:48:27,886 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:27,886 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-12T06:48:27,887 INFO [StoreOpener-abf9890bb5fa9b12fd7d1c33ce62b875-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:27,888 INFO [StoreOpener-abf9890bb5fa9b12fd7d1c33ce62b875-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:48:27,888 INFO [StoreOpener-abf9890bb5fa9b12fd7d1c33ce62b875-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abf9890bb5fa9b12fd7d1c33ce62b875 columnFamilyName A 2024-11-12T06:48:27,888 DEBUG [StoreOpener-abf9890bb5fa9b12fd7d1c33ce62b875-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:27,889 INFO [StoreOpener-abf9890bb5fa9b12fd7d1c33ce62b875-1 {}] regionserver.HStore(327): Store=abf9890bb5fa9b12fd7d1c33ce62b875/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:48:27,889 INFO [StoreOpener-abf9890bb5fa9b12fd7d1c33ce62b875-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:27,890 INFO [StoreOpener-abf9890bb5fa9b12fd7d1c33ce62b875-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:48:27,890 INFO [StoreOpener-abf9890bb5fa9b12fd7d1c33ce62b875-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abf9890bb5fa9b12fd7d1c33ce62b875 columnFamilyName B 2024-11-12T06:48:27,890 DEBUG [StoreOpener-abf9890bb5fa9b12fd7d1c33ce62b875-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:27,891 INFO [StoreOpener-abf9890bb5fa9b12fd7d1c33ce62b875-1 {}] regionserver.HStore(327): Store=abf9890bb5fa9b12fd7d1c33ce62b875/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:48:27,891 INFO [StoreOpener-abf9890bb5fa9b12fd7d1c33ce62b875-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:27,892 INFO [StoreOpener-abf9890bb5fa9b12fd7d1c33ce62b875-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:48:27,892 INFO [StoreOpener-abf9890bb5fa9b12fd7d1c33ce62b875-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abf9890bb5fa9b12fd7d1c33ce62b875 columnFamilyName C 2024-11-12T06:48:27,892 DEBUG [StoreOpener-abf9890bb5fa9b12fd7d1c33ce62b875-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:27,892 INFO [StoreOpener-abf9890bb5fa9b12fd7d1c33ce62b875-1 {}] regionserver.HStore(327): Store=abf9890bb5fa9b12fd7d1c33ce62b875/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:48:27,892 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:27,892 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:27,893 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:27,893 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:27,894 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:27,894 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:27,895 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T06:48:27,896 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:27,898 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T06:48:27,898 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened abf9890bb5fa9b12fd7d1c33ce62b875; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64857260, jitterRate=-0.03355151414871216}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T06:48:27,898 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:27,899 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for abf9890bb5fa9b12fd7d1c33ce62b875: Running coprocessor pre-open hook at 1731394107886Writing region info on filesystem at 1731394107886Initializing all the Stores at 1731394107886Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394107886Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394107887 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394107887Cleaning up temporary data from old regions at 1731394107894 (+7 ms)Running coprocessor post-open hooks at 1731394107898 (+4 ms)Region opened successfully at 1731394107899 (+1 ms) 2024-11-12T06:48:27,900 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., pid=63, masterSystemTime=1731394107881 2024-11-12T06:48:27,902 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:27,902 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:27,902 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=abf9890bb5fa9b12fd7d1c33ce62b875, regionState=OPEN, openSeqNum=2, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:27,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 because future has completed 2024-11-12T06:48:27,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-12T06:48:27,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 in 175 msec 2024-11-12T06:48:27,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-12T06:48:27,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=abf9890bb5fa9b12fd7d1c33ce62b875, ASSIGN in 331 msec 2024-11-12T06:48:27,910 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T06:48:27,910 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394107910"}]},"ts":"1731394107910"} 2024-11-12T06:48:27,912 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T06:48:27,913 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T06:48:27,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-11-12T06:48:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-12T06:48:28,897 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:28,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,899 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:28,899 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:28,900 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:28,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67971bf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:28,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:28,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:28,902 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:28,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5275516e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,903 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:28,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:28,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:28,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:28,907 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T06:48:28,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 9f0beee3d6b6,43031,1731394046864 2024-11-12T06:48:28,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@701df2f4 2024-11-12T06:48:28,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T06:48:28,908 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T06:48:28,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,909 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:28,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:28,910 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:28,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:28,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:28,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@529d7f8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:28,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:28,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:28,911 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:28,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78f284d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,913 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:28,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:28,913 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:28,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:28,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:28,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b2e17f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:28,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:28,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:28,915 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:28,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73d75a21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,916 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:28,916 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:28,916 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62aa3cd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:28,918 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17c130b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,919 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:28,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:28,919 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@686ce59f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:28,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:28,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:28,920 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:28,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14d723ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,922 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:28,922 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:28,922 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:28,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:28,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:28,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54bf9417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:28,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:28,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:28,924 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:28,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302de99c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:28,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:28,926 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:28,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:28,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:28,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d895511, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:28,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:28,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:28,927 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63b5c8b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,928 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:28,928 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:28,929 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:28,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:28,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:28,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625ea99d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:28,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:28,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:28,930 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12ace379, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,931 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:28,931 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:28,932 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:28,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:28,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:28,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37dff720, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:28,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:28,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:28,933 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:28,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b0e2e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:28,935 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:28,935 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1440af2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:28,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:28,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:28,937 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:28,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33b9d8d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,938 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:28,938 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:28,939 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:28,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:28,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:28,940 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7732c9a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,940 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:28,940 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:28,940 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:28,941 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2efdb53f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:28,942 DEBUG [Thread-1088 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:28,942 DEBUG [Thread-1095 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:28,942 DEBUG [Thread-1097 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:28,942 DEBUG [Thread-1101 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:28,942 DEBUG [Thread-1103 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:28,942 DEBUG [Thread-1084 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:28,942 DEBUG [Thread-1086 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:28,942 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:48:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:28,943 DEBUG [Thread-1097 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:28,943 DEBUG [Thread-1103 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:28,943 DEBUG [Thread-1097 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:28,943 DEBUG [Thread-1103 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:28,944 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:28,944 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:28,944 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:28,944 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:28,944 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:28,945 DEBUG [Thread-1092 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:28,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:28,946 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:28,946 DEBUG [Thread-1090 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:28,946 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:28,946 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:28,947 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:28,947 DEBUG [Thread-1090 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:28,947 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:28,947 DEBUG [Thread-1099 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:28,947 DEBUG [Thread-1090 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-12T06:48:28,948 DEBUG [Thread-1099 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:28,948 DEBUG [Thread-1099 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:28,949 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:28,949 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:48:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:48:28,949 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:48:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:48:28,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:48:28,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:48:28,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:48:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T06:48:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:48:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:48:28,952 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:48:28,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:28,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:48:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:48:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:48:28,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:28,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing abf9890bb5fa9b12fd7d1c33ce62b875 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T06:48:28,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=A 2024-11-12T06:48:28,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:28,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=B 2024-11-12T06:48:28,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:28,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=C 2024-11-12T06:48:28,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:28,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394168995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394168997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394168998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394168999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/dbc6fdb2cfc641039aed00a8c8314886 is 50, key is test_row_0/A:col10/1731394108967/Put/seqid=0 2024-11-12T06:48:29,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394169010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742057_1233 (size=12001) 2024-11-12T06:48:29,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/dbc6fdb2cfc641039aed00a8c8314886 2024-11-12T06:48:29,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/b958513f0367416b99065a8181753372 is 50, key is test_row_0/B:col10/1731394108967/Put/seqid=0 2024-11-12T06:48:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T06:48:29,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742058_1234 (size=12001) 2024-11-12T06:48:29,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T06:48:29,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:29,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:29,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:29,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:29,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:29,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394169107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394169107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394169107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394169107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394169117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T06:48:29,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:29,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:29,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:29,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:29,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T06:48:29,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394169317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394169317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394169317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394169317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394169327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T06:48:29,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:29,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:29,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:29,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:29,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:29,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/b958513f0367416b99065a8181753372 2024-11-12T06:48:29,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/48295ba63bce44dca13b97c9a748667d is 50, key is test_row_0/C:col10/1731394108967/Put/seqid=0 2024-11-12T06:48:29,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742059_1235 (size=12001) 2024-11-12T06:48:29,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/48295ba63bce44dca13b97c9a748667d 2024-11-12T06:48:29,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/dbc6fdb2cfc641039aed00a8c8314886 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/dbc6fdb2cfc641039aed00a8c8314886 2024-11-12T06:48:29,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/dbc6fdb2cfc641039aed00a8c8314886, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T06:48:29,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/b958513f0367416b99065a8181753372 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/b958513f0367416b99065a8181753372 2024-11-12T06:48:29,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/b958513f0367416b99065a8181753372, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T06:48:29,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/48295ba63bce44dca13b97c9a748667d as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/48295ba63bce44dca13b97c9a748667d 2024-11-12T06:48:29,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/48295ba63bce44dca13b97c9a748667d, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T06:48:29,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for abf9890bb5fa9b12fd7d1c33ce62b875 in 558ms, sequenceid=14, compaction requested=false 2024-11-12T06:48:29,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:29,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T06:48:29,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:29,564 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing abf9890bb5fa9b12fd7d1c33ce62b875 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T06:48:29,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=A 2024-11-12T06:48:29,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:29,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=B 2024-11-12T06:48:29,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:29,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=C 2024-11-12T06:48:29,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:29,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/0c4c9d2c4b5b4d18aae3493529435f9b is 50, key is test_row_0/A:col10/1731394108996/Put/seqid=0 2024-11-12T06:48:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T06:48:29,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742060_1236 (size=12001) 2024-11-12T06:48:29,585 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/0c4c9d2c4b5b4d18aae3493529435f9b 2024-11-12T06:48:29,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/9f3d9000e72c411b8d54f8e662c37b4c is 50, key is test_row_0/B:col10/1731394108996/Put/seqid=0 2024-11-12T06:48:29,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742061_1237 (size=12001) 2024-11-12T06:48:29,619 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/9f3d9000e72c411b8d54f8e662c37b4c 2024-11-12T06:48:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:29,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:29,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/29d8003dce8e4c4fbe16625d0e15c02b is 50, key is test_row_0/C:col10/1731394108996/Put/seqid=0 2024-11-12T06:48:29,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394169638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394169639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394169648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394169648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742062_1238 (size=12001) 2024-11-12T06:48:29,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,654 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/29d8003dce8e4c4fbe16625d0e15c02b 2024-11-12T06:48:29,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394169648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/0c4c9d2c4b5b4d18aae3493529435f9b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/0c4c9d2c4b5b4d18aae3493529435f9b 2024-11-12T06:48:29,665 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/0c4c9d2c4b5b4d18aae3493529435f9b, entries=150, sequenceid=33, filesize=11.7 K 2024-11-12T06:48:29,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/9f3d9000e72c411b8d54f8e662c37b4c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/9f3d9000e72c411b8d54f8e662c37b4c 2024-11-12T06:48:29,672 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/9f3d9000e72c411b8d54f8e662c37b4c, entries=150, sequenceid=33, filesize=11.7 K 2024-11-12T06:48:29,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/29d8003dce8e4c4fbe16625d0e15c02b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/29d8003dce8e4c4fbe16625d0e15c02b 2024-11-12T06:48:29,677 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/29d8003dce8e4c4fbe16625d0e15c02b, entries=150, sequenceid=33, filesize=11.7 K 2024-11-12T06:48:29,678 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for abf9890bb5fa9b12fd7d1c33ce62b875 in 114ms, sequenceid=33, compaction requested=false 2024-11-12T06:48:29,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:29,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:29,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-12T06:48:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-12T06:48:29,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-12T06:48:29,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 727 msec 2024-11-12T06:48:29,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 740 msec 2024-11-12T06:48:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:29,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing abf9890bb5fa9b12fd7d1c33ce62b875 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T06:48:29,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=A 2024-11-12T06:48:29,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:29,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=B 2024-11-12T06:48:29,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:29,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=C 2024-11-12T06:48:29,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:29,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/c4c26e843e30476da167a6ceff0e00d7 is 50, key is test_row_0/A:col10/1731394109760/Put/seqid=0 2024-11-12T06:48:29,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394169773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394169774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394169775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394169776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394169777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742063_1239 (size=12001) 2024-11-12T06:48:29,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394169887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394169887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394169887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394169887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:29,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:29,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394169888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:29,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:29,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:30,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T06:48:30,087 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:30,088 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:48:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-12T06:48:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T06:48:30,089 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:48:30,090 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:48:30,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:48:30,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:30,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394170097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394170097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:30,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394170098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394170098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394170098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:30,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/c4c26e843e30476da167a6ceff0e00d7 2024-11-12T06:48:30,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T06:48:30,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/9032ff40632541bdbb8eeaf213bab2ce is 50, key is test_row_0/B:col10/1731394109760/Put/seqid=0 2024-11-12T06:48:30,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742064_1240 (size=12001) 2024-11-12T06:48:30,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T06:48:30,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:30,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:30,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:30,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:30,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:30,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T06:48:30,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:30,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:30,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:30,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:30,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:30,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T06:48:30,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394170407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:30,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394170407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:30,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394170407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:30,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:30,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:30,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394170407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:30,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:30,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:30,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394170407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:30,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:30,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T06:48:30,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:30,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:30,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:30,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:30,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:30,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:30,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/9032ff40632541bdbb8eeaf213bab2ce 2024-11-12T06:48:30,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/5c735e4bf6ac42fe9edffac342002b50 is 50, key is test_row_0/C:col10/1731394109760/Put/seqid=0 2024-11-12T06:48:30,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742065_1241 (size=12001) 2024-11-12T06:48:30,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T06:48:30,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:30,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:30,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:30,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:30,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:30,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T06:48:30,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T06:48:30,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:30,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:30,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:30,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:30,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:30,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:30,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394170917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:30,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:30,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:30,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394170917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:30,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:30,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:30,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394170917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:30,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:30,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:30,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394170918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:30,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:30,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:30,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394170918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:30,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:30,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:31,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T06:48:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:31,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:31,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:31,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/5c735e4bf6ac42fe9edffac342002b50 2024-11-12T06:48:31,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/c4c26e843e30476da167a6ceff0e00d7 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/c4c26e843e30476da167a6ceff0e00d7 2024-11-12T06:48:31,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/c4c26e843e30476da167a6ceff0e00d7, entries=150, sequenceid=50, filesize=11.7 K 2024-11-12T06:48:31,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/9032ff40632541bdbb8eeaf213bab2ce as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/9032ff40632541bdbb8eeaf213bab2ce 2024-11-12T06:48:31,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/9032ff40632541bdbb8eeaf213bab2ce, entries=150, sequenceid=50, filesize=11.7 K 2024-11-12T06:48:31,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/5c735e4bf6ac42fe9edffac342002b50 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/5c735e4bf6ac42fe9edffac342002b50 2024-11-12T06:48:31,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/5c735e4bf6ac42fe9edffac342002b50, entries=150, sequenceid=50, filesize=11.7 K 2024-11-12T06:48:31,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for abf9890bb5fa9b12fd7d1c33ce62b875 in 1280ms, sequenceid=50, compaction requested=true 2024-11-12T06:48:31,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:31,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:48:31,041 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:31,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:31,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:48:31,041 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:31,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:31,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:48:31,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:31,042 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:31,042 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:31,042 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/B is initiating minor compaction (all files) 2024-11-12T06:48:31,042 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/A is initiating minor compaction (all files) 2024-11-12T06:48:31,042 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/B in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:31,042 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/A in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:31,042 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/b958513f0367416b99065a8181753372, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/9f3d9000e72c411b8d54f8e662c37b4c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/9032ff40632541bdbb8eeaf213bab2ce] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=35.2 K 2024-11-12T06:48:31,042 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/dbc6fdb2cfc641039aed00a8c8314886, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/0c4c9d2c4b5b4d18aae3493529435f9b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/c4c26e843e30476da167a6ceff0e00d7] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=35.2 K 2024-11-12T06:48:31,042 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting b958513f0367416b99065a8181753372, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731394108965 2024-11-12T06:48:31,042 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbc6fdb2cfc641039aed00a8c8314886, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731394108965 2024-11-12T06:48:31,043 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f3d9000e72c411b8d54f8e662c37b4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731394108983 2024-11-12T06:48:31,043 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c4c9d2c4b5b4d18aae3493529435f9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731394108983 2024-11-12T06:48:31,043 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 9032ff40632541bdbb8eeaf213bab2ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731394109630 2024-11-12T06:48:31,043 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4c26e843e30476da167a6ceff0e00d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731394109630 2024-11-12T06:48:31,050 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#A#compaction#192 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:31,051 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#B#compaction#193 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:31,051 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/d5312f27e25a4ade9e7af8d2c1535907 is 50, key is test_row_0/A:col10/1731394109760/Put/seqid=0 2024-11-12T06:48:31,051 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/e7edc347256f4f1ba3d5ef0818799f6b is 50, key is test_row_0/B:col10/1731394109760/Put/seqid=0 2024-11-12T06:48:31,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742067_1243 (size=12104) 2024-11-12T06:48:31,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742066_1242 (size=12104) 2024-11-12T06:48:31,083 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/e7edc347256f4f1ba3d5ef0818799f6b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/e7edc347256f4f1ba3d5ef0818799f6b 2024-11-12T06:48:31,084 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/d5312f27e25a4ade9e7af8d2c1535907 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/d5312f27e25a4ade9e7af8d2c1535907 2024-11-12T06:48:31,089 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/B of abf9890bb5fa9b12fd7d1c33ce62b875 into e7edc347256f4f1ba3d5ef0818799f6b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:31,089 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:31,089 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/B, priority=13, startTime=1731394111041; duration=0sec 2024-11-12T06:48:31,089 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:31,089 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:B 2024-11-12T06:48:31,089 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:31,091 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/A of abf9890bb5fa9b12fd7d1c33ce62b875 into d5312f27e25a4ade9e7af8d2c1535907(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:31,091 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:31,091 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/A, priority=13, startTime=1731394111041; duration=0sec 2024-11-12T06:48:31,091 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:31,091 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:A 2024-11-12T06:48:31,091 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:31,091 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/C is initiating minor compaction (all files) 2024-11-12T06:48:31,091 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/C in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:31,092 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/48295ba63bce44dca13b97c9a748667d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/29d8003dce8e4c4fbe16625d0e15c02b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/5c735e4bf6ac42fe9edffac342002b50] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=35.2 K 2024-11-12T06:48:31,092 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 48295ba63bce44dca13b97c9a748667d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731394108965 2024-11-12T06:48:31,092 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 29d8003dce8e4c4fbe16625d0e15c02b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731394108983 2024-11-12T06:48:31,092 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c735e4bf6ac42fe9edffac342002b50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731394109630 2024-11-12T06:48:31,100 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#C#compaction#194 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:31,100 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/a56c503443de46e7acbb03a383a8735c is 50, key is test_row_0/C:col10/1731394109760/Put/seqid=0 2024-11-12T06:48:31,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742068_1244 (size=12104) 2024-11-12T06:48:31,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T06:48:31,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:31,161 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing abf9890bb5fa9b12fd7d1c33ce62b875 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T06:48:31,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=A 2024-11-12T06:48:31,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:31,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=B 2024-11-12T06:48:31,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:31,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=C 2024-11-12T06:48:31,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:31,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/1f3806be1e1e4c9882d7b90347026d9b is 50, key is test_row_0/A:col10/1731394109774/Put/seqid=0 2024-11-12T06:48:31,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742069_1245 (size=12001) 2024-11-12T06:48:31,171 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/1f3806be1e1e4c9882d7b90347026d9b 2024-11-12T06:48:31,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/d7a4b81cf24d4839a908a98a9c7b008c is 50, key is test_row_0/B:col10/1731394109774/Put/seqid=0 2024-11-12T06:48:31,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742070_1246 (size=12001) 2024-11-12T06:48:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T06:48:31,513 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/a56c503443de46e7acbb03a383a8735c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/a56c503443de46e7acbb03a383a8735c 2024-11-12T06:48:31,517 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/C of abf9890bb5fa9b12fd7d1c33ce62b875 into a56c503443de46e7acbb03a383a8735c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:31,517 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:31,517 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/C, priority=13, startTime=1731394111041; duration=0sec 2024-11-12T06:48:31,517 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:31,517 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:C 2024-11-12T06:48:31,583 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/d7a4b81cf24d4839a908a98a9c7b008c 2024-11-12T06:48:31,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/ae019e9a739d4f76bb2a0d4a24c296eb is 50, key is test_row_0/C:col10/1731394109774/Put/seqid=0 2024-11-12T06:48:31,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742071_1247 (size=12001) 2024-11-12T06:48:31,594 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/ae019e9a739d4f76bb2a0d4a24c296eb 2024-11-12T06:48:31,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/1f3806be1e1e4c9882d7b90347026d9b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/1f3806be1e1e4c9882d7b90347026d9b 2024-11-12T06:48:31,603 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/1f3806be1e1e4c9882d7b90347026d9b, entries=150, sequenceid=67, filesize=11.7 K 2024-11-12T06:48:31,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/d7a4b81cf24d4839a908a98a9c7b008c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/d7a4b81cf24d4839a908a98a9c7b008c 2024-11-12T06:48:31,608 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/d7a4b81cf24d4839a908a98a9c7b008c, entries=150, sequenceid=67, filesize=11.7 K 2024-11-12T06:48:31,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/ae019e9a739d4f76bb2a0d4a24c296eb as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ae019e9a739d4f76bb2a0d4a24c296eb 2024-11-12T06:48:31,614 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ae019e9a739d4f76bb2a0d4a24c296eb, entries=150, sequenceid=67, filesize=11.7 K 2024-11-12T06:48:31,616 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for abf9890bb5fa9b12fd7d1c33ce62b875 in 455ms, sequenceid=67, compaction requested=false 2024-11-12T06:48:31,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:31,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:31,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-12T06:48:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-12T06:48:31,619 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-12T06:48:31,619 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5270 sec 2024-11-12T06:48:31,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.5310 sec 2024-11-12T06:48:31,938 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T06:48:31,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:31,939 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing abf9890bb5fa9b12fd7d1c33ce62b875 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T06:48:31,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=A 2024-11-12T06:48:31,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:31,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=B 2024-11-12T06:48:31,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:31,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=C 2024-11-12T06:48:31,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:31,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/266cc74af26b4d2cba078a76b1416ae0 is 50, key is test_row_0/A:col10/1731394111939/Put/seqid=0 2024-11-12T06:48:31,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742072_1248 (size=14341) 2024-11-12T06:48:31,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:31,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:31,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394171969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:31,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394171969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:31,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:31,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394171970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:31,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:31,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:31,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:31,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:31,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394171970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:31,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:31,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:31,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:31,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:31,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:31,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:31,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:31,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:31,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:31,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:31,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394171974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:31,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:31,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:31,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:32,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394172087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:32,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:32,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394172088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:32,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:32,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:32,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394172088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:32,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:32,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:32,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394172088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:32,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394172088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:32,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:32,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T06:48:32,237 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:32,237 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:48:32,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-12T06:48:32,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T06:48:32,239 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:48:32,239 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:48:32,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:48:32,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:32,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394172297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:32,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394172297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:32,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:32,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394172298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394172298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394172298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:32,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T06:48:32,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/266cc74af26b4d2cba078a76b1416ae0 2024-11-12T06:48:32,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/ea9b076238bf41a2a0c17dedb9e308f9 is 50, key is test_row_0/B:col10/1731394111939/Put/seqid=0 2024-11-12T06:48:32,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742073_1249 (size=12001) 2024-11-12T06:48:32,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T06:48:32,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:32,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:32,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:32,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:32,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:32,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:32,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T06:48:32,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:32,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:32,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:32,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:32,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:32,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:32,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T06:48:32,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:32,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394172607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:32,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:32,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:32,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394172608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:32,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:32,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394172608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:32,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:32,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394172608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:32,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:32,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394172608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:32,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T06:48:32,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:32,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:32,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:32,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:32,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:32,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/ea9b076238bf41a2a0c17dedb9e308f9 2024-11-12T06:48:32,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/95b2cb9068bb428c834d48dbd4667c4b is 50, key is test_row_0/C:col10/1731394111939/Put/seqid=0 2024-11-12T06:48:32,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742074_1250 (size=12001) 2024-11-12T06:48:32,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T06:48:32,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:32,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:32,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:32,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:32,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:32,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T06:48:33,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T06:48:33,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:33,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:33,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:33,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:33,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:33,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:33,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394173117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:33,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:33,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:33,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:33,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:33,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394173117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:33,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:33,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:33,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:33,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:33,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394173117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:33,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:33,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:33,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:33,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394173117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:33,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:33,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394173118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:33,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T06:48:33,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:33,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:33,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:33,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:33,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:33,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:33,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/95b2cb9068bb428c834d48dbd4667c4b 2024-11-12T06:48:33,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/266cc74af26b4d2cba078a76b1416ae0 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/266cc74af26b4d2cba078a76b1416ae0 2024-11-12T06:48:33,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/266cc74af26b4d2cba078a76b1416ae0, entries=200, sequenceid=80, filesize=14.0 K 2024-11-12T06:48:33,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/ea9b076238bf41a2a0c17dedb9e308f9 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/ea9b076238bf41a2a0c17dedb9e308f9 2024-11-12T06:48:33,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/ea9b076238bf41a2a0c17dedb9e308f9, entries=150, sequenceid=80, filesize=11.7 K 2024-11-12T06:48:33,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/95b2cb9068bb428c834d48dbd4667c4b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/95b2cb9068bb428c834d48dbd4667c4b 2024-11-12T06:48:33,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/95b2cb9068bb428c834d48dbd4667c4b, entries=150, sequenceid=80, filesize=11.7 K 2024-11-12T06:48:33,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for abf9890bb5fa9b12fd7d1c33ce62b875 in 1267ms, sequenceid=80, compaction requested=true 2024-11-12T06:48:33,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:33,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:48:33,206 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:33,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:33,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:48:33,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:33,206 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:33,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:48:33,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:33,207 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:33,207 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:33,207 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/B is initiating minor compaction (all files) 2024-11-12T06:48:33,207 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/A is initiating minor compaction (all files) 2024-11-12T06:48:33,207 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/B in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:33,207 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/A in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:33,208 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/e7edc347256f4f1ba3d5ef0818799f6b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/d7a4b81cf24d4839a908a98a9c7b008c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/ea9b076238bf41a2a0c17dedb9e308f9] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=35.3 K 2024-11-12T06:48:33,208 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/d5312f27e25a4ade9e7af8d2c1535907, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/1f3806be1e1e4c9882d7b90347026d9b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/266cc74af26b4d2cba078a76b1416ae0] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=37.5 K 2024-11-12T06:48:33,208 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5312f27e25a4ade9e7af8d2c1535907, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731394109630 2024-11-12T06:48:33,208 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting e7edc347256f4f1ba3d5ef0818799f6b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731394109630 2024-11-12T06:48:33,209 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f3806be1e1e4c9882d7b90347026d9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731394109772 2024-11-12T06:48:33,209 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting d7a4b81cf24d4839a908a98a9c7b008c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731394109772 2024-11-12T06:48:33,209 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting ea9b076238bf41a2a0c17dedb9e308f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731394111938 2024-11-12T06:48:33,209 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 266cc74af26b4d2cba078a76b1416ae0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731394111937 2024-11-12T06:48:33,218 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#B#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:33,219 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#A#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:33,219 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/21973ceb9e0f49718209fd2301aafc79 is 50, key is test_row_0/B:col10/1731394111939/Put/seqid=0 2024-11-12T06:48:33,219 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/67cc1cde43aa41888653d2738c4b1a53 is 50, key is test_row_0/A:col10/1731394111939/Put/seqid=0 2024-11-12T06:48:33,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742075_1251 (size=12207) 2024-11-12T06:48:33,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742076_1252 (size=12207) 2024-11-12T06:48:33,234 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/21973ceb9e0f49718209fd2301aafc79 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/21973ceb9e0f49718209fd2301aafc79 2024-11-12T06:48:33,239 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/B of abf9890bb5fa9b12fd7d1c33ce62b875 into 21973ceb9e0f49718209fd2301aafc79(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:33,239 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:33,239 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/B, priority=13, startTime=1731394113206; duration=0sec 2024-11-12T06:48:33,239 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:33,239 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:B 2024-11-12T06:48:33,239 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:33,241 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:33,241 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/C is initiating minor compaction (all files) 2024-11-12T06:48:33,241 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/C in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:33,241 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/a56c503443de46e7acbb03a383a8735c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ae019e9a739d4f76bb2a0d4a24c296eb, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/95b2cb9068bb428c834d48dbd4667c4b] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=35.3 K 2024-11-12T06:48:33,241 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting a56c503443de46e7acbb03a383a8735c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731394109630 2024-11-12T06:48:33,242 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting ae019e9a739d4f76bb2a0d4a24c296eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731394109772 2024-11-12T06:48:33,242 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 95b2cb9068bb428c834d48dbd4667c4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731394111938 2024-11-12T06:48:33,251 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#C#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:33,252 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/fe83f5f8acca4d8ba30b0731b4beecf9 is 50, key is test_row_0/C:col10/1731394111939/Put/seqid=0 2024-11-12T06:48:33,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742077_1253 (size=12207) 2024-11-12T06:48:33,264 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/fe83f5f8acca4d8ba30b0731b4beecf9 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/fe83f5f8acca4d8ba30b0731b4beecf9 2024-11-12T06:48:33,268 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/C of abf9890bb5fa9b12fd7d1c33ce62b875 into fe83f5f8acca4d8ba30b0731b4beecf9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:33,268 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:33,268 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/C, priority=13, startTime=1731394113206; duration=0sec 2024-11-12T06:48:33,268 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:33,268 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:C 2024-11-12T06:48:33,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T06:48:33,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:33,307 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing abf9890bb5fa9b12fd7d1c33ce62b875 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T06:48:33,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=A 2024-11-12T06:48:33,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:33,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=B 2024-11-12T06:48:33,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:33,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=C 2024-11-12T06:48:33,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:33,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/d62df8c78585496f8b703744fd7f4fa6 is 50, key is test_row_0/A:col10/1731394111969/Put/seqid=0 2024-11-12T06:48:33,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742078_1254 (size=12001) 2024-11-12T06:48:33,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T06:48:33,634 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/67cc1cde43aa41888653d2738c4b1a53 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/67cc1cde43aa41888653d2738c4b1a53 2024-11-12T06:48:33,639 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/A of abf9890bb5fa9b12fd7d1c33ce62b875 into 67cc1cde43aa41888653d2738c4b1a53(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:33,639 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:33,639 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/A, priority=13, startTime=1731394113206; duration=0sec 2024-11-12T06:48:33,639 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:33,639 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:A 2024-11-12T06:48:33,717 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/d62df8c78585496f8b703744fd7f4fa6 2024-11-12T06:48:33,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/017c3570ea654bca8f5690cca712a40e is 50, key is test_row_0/B:col10/1731394111969/Put/seqid=0 2024-11-12T06:48:33,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742079_1255 (size=12001) 2024-11-12T06:48:34,129 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/017c3570ea654bca8f5690cca712a40e 2024-11-12T06:48:34,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:34,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:34,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/3eaf431f32b44fab8de34e708e830ad7 is 50, key is test_row_0/C:col10/1731394111969/Put/seqid=0 2024-11-12T06:48:34,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394174134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394174136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394174137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394174139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394174139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742080_1256 (size=12001) 2024-11-12T06:48:34,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394174247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394174247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394174247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394174248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394174248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T06:48:34,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394174457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394174457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394174458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394174458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394174458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,540 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/3eaf431f32b44fab8de34e708e830ad7 2024-11-12T06:48:34,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/d62df8c78585496f8b703744fd7f4fa6 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/d62df8c78585496f8b703744fd7f4fa6 2024-11-12T06:48:34,549 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/d62df8c78585496f8b703744fd7f4fa6, entries=150, sequenceid=102, filesize=11.7 K 2024-11-12T06:48:34,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/017c3570ea654bca8f5690cca712a40e as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/017c3570ea654bca8f5690cca712a40e 2024-11-12T06:48:34,564 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/017c3570ea654bca8f5690cca712a40e, entries=150, sequenceid=102, filesize=11.7 K 2024-11-12T06:48:34,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/3eaf431f32b44fab8de34e708e830ad7 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/3eaf431f32b44fab8de34e708e830ad7 2024-11-12T06:48:34,569 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/3eaf431f32b44fab8de34e708e830ad7, entries=150, sequenceid=102, filesize=11.7 K 2024-11-12T06:48:34,570 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for abf9890bb5fa9b12fd7d1c33ce62b875 in 1262ms, sequenceid=102, compaction requested=false 2024-11-12T06:48:34,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:34,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:34,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-12T06:48:34,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-12T06:48:34,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-12T06:48:34,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3340 sec 2024-11-12T06:48:34,577 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.3390 sec 2024-11-12T06:48:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:34,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing abf9890bb5fa9b12fd7d1c33ce62b875 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T06:48:34,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=A 2024-11-12T06:48:34,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:34,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=B 2024-11-12T06:48:34,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:34,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=C 2024-11-12T06:48:34,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:34,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/bc8a3d093da14f8e85324c0b56ec4f9b is 50, key is test_row_0/A:col10/1731394114770/Put/seqid=0 2024-11-12T06:48:34,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742081_1257 (size=16681) 2024-11-12T06:48:34,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/bc8a3d093da14f8e85324c0b56ec4f9b 2024-11-12T06:48:34,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394174785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394174786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394174787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394174789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/9df3092c249e42bb852d3b8134c2a04b is 50, key is test_row_0/B:col10/1731394114770/Put/seqid=0 2024-11-12T06:48:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394174789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742082_1258 (size=12001) 2024-11-12T06:48:34,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/9df3092c249e42bb852d3b8134c2a04b 2024-11-12T06:48:34,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/e9b1957e32b1498f9a6963587c371d44 is 50, key is test_row_0/C:col10/1731394114770/Put/seqid=0 2024-11-12T06:48:34,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742083_1259 (size=12001) 2024-11-12T06:48:34,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394174897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394174897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394174897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394174897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394174897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:35,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394175107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:35,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:35,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394175107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:35,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:35,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394175107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:35,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394175107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394175107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:35,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/e9b1957e32b1498f9a6963587c371d44 2024-11-12T06:48:35,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/bc8a3d093da14f8e85324c0b56ec4f9b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/bc8a3d093da14f8e85324c0b56ec4f9b 2024-11-12T06:48:35,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/bc8a3d093da14f8e85324c0b56ec4f9b, entries=250, sequenceid=121, filesize=16.3 K 2024-11-12T06:48:35,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/9df3092c249e42bb852d3b8134c2a04b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/9df3092c249e42bb852d3b8134c2a04b 2024-11-12T06:48:35,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/9df3092c249e42bb852d3b8134c2a04b, entries=150, sequenceid=121, filesize=11.7 K 2024-11-12T06:48:35,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/e9b1957e32b1498f9a6963587c371d44 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/e9b1957e32b1498f9a6963587c371d44 2024-11-12T06:48:35,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/e9b1957e32b1498f9a6963587c371d44, entries=150, sequenceid=121, filesize=11.7 K 2024-11-12T06:48:35,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for abf9890bb5fa9b12fd7d1c33ce62b875 in 501ms, sequenceid=121, compaction requested=true 2024-11-12T06:48:35,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:35,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:48:35,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:35,272 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:35,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:48:35,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:35,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:48:35,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:35,272 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:35,273 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:35,273 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:35,273 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/A is initiating minor compaction (all files) 2024-11-12T06:48:35,273 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/B is initiating minor compaction (all files) 2024-11-12T06:48:35,273 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/A in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:35,273 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/B in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:35,273 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/67cc1cde43aa41888653d2738c4b1a53, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/d62df8c78585496f8b703744fd7f4fa6, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/bc8a3d093da14f8e85324c0b56ec4f9b] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=39.9 K 2024-11-12T06:48:35,273 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/21973ceb9e0f49718209fd2301aafc79, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/017c3570ea654bca8f5690cca712a40e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/9df3092c249e42bb852d3b8134c2a04b] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=35.4 K 2024-11-12T06:48:35,274 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 21973ceb9e0f49718209fd2301aafc79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731394111938 2024-11-12T06:48:35,274 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67cc1cde43aa41888653d2738c4b1a53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731394111938 2024-11-12T06:48:35,275 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 017c3570ea654bca8f5690cca712a40e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731394111956 2024-11-12T06:48:35,275 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting d62df8c78585496f8b703744fd7f4fa6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731394111956 2024-11-12T06:48:35,275 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 9df3092c249e42bb852d3b8134c2a04b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731394114136 2024-11-12T06:48:35,275 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc8a3d093da14f8e85324c0b56ec4f9b, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731394114136 2024-11-12T06:48:35,283 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#B#compaction#211 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:35,283 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#A#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:35,283 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/a1ff2ce4c8994712937e6fb343b5e55c is 50, key is test_row_0/A:col10/1731394114770/Put/seqid=0 2024-11-12T06:48:35,283 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/6d966b9d979846b9bee9a071ea2c819f is 50, key is test_row_0/B:col10/1731394114770/Put/seqid=0 2024-11-12T06:48:35,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742085_1261 (size=12309) 2024-11-12T06:48:35,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742084_1260 (size=12309) 2024-11-12T06:48:35,301 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/6d966b9d979846b9bee9a071ea2c819f as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/6d966b9d979846b9bee9a071ea2c819f 2024-11-12T06:48:35,306 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/B of abf9890bb5fa9b12fd7d1c33ce62b875 into 6d966b9d979846b9bee9a071ea2c819f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:35,306 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:35,306 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/B, priority=13, startTime=1731394115272; duration=0sec 2024-11-12T06:48:35,307 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:35,307 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:B 2024-11-12T06:48:35,307 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:35,308 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:35,308 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/C is initiating minor compaction (all files) 2024-11-12T06:48:35,308 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/C in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:35,308 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/fe83f5f8acca4d8ba30b0731b4beecf9, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/3eaf431f32b44fab8de34e708e830ad7, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/e9b1957e32b1498f9a6963587c371d44] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=35.4 K 2024-11-12T06:48:35,309 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting fe83f5f8acca4d8ba30b0731b4beecf9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731394111938 2024-11-12T06:48:35,309 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eaf431f32b44fab8de34e708e830ad7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731394111956 2024-11-12T06:48:35,309 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting e9b1957e32b1498f9a6963587c371d44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731394114136 2024-11-12T06:48:35,316 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#C#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:35,317 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/ca273260399d4a9d922ed852527ebcae is 50, key is test_row_0/C:col10/1731394114770/Put/seqid=0 2024-11-12T06:48:35,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742086_1262 (size=12309) 2024-11-12T06:48:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:35,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing abf9890bb5fa9b12fd7d1c33ce62b875 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T06:48:35,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=A 2024-11-12T06:48:35,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:35,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=B 2024-11-12T06:48:35,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:35,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=C 2024-11-12T06:48:35,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:35,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/944a414fc7674f3ea8a8ea57538c09d4 is 50, key is test_row_0/A:col10/1731394115419/Put/seqid=0 2024-11-12T06:48:35,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742087_1263 (size=12151) 2024-11-12T06:48:35,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/944a414fc7674f3ea8a8ea57538c09d4 2024-11-12T06:48:35,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/4b897cb89cc44a4ab7b1d137e2947949 is 50, key is test_row_0/B:col10/1731394115419/Put/seqid=0 2024-11-12T06:48:35,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:35,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394175450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:35,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:35,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742088_1264 (size=12151) 2024-11-12T06:48:35,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394175452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:35,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:35,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394175454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394175455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:35,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394175455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:35,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394175557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:35,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:35,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394175567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:35,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:35,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394175567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:35,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:35,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:35,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394175567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:35,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:35,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394175567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:35,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:35,697 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/a1ff2ce4c8994712937e6fb343b5e55c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/a1ff2ce4c8994712937e6fb343b5e55c 2024-11-12T06:48:35,701 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/A of abf9890bb5fa9b12fd7d1c33ce62b875 into a1ff2ce4c8994712937e6fb343b5e55c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:35,701 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:35,701 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/A, priority=13, startTime=1731394115272; duration=0sec 2024-11-12T06:48:35,701 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:35,701 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:A 2024-11-12T06:48:35,726 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/ca273260399d4a9d922ed852527ebcae as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ca273260399d4a9d922ed852527ebcae 2024-11-12T06:48:35,733 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/C of abf9890bb5fa9b12fd7d1c33ce62b875 into ca273260399d4a9d922ed852527ebcae(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:35,733 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:35,733 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/C, priority=13, startTime=1731394115272; duration=0sec 2024-11-12T06:48:35,733 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:35,733 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:C 2024-11-12T06:48:35,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394175767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:35,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:35,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394175777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:35,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:35,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394175777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:35,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:35,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394175777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:35,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:35,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394175777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:35,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:35,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:35,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/4b897cb89cc44a4ab7b1d137e2947949 2024-11-12T06:48:35,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/ad91a8a1ca62452c87d8604123595a6c is 50, key is test_row_0/C:col10/1731394115419/Put/seqid=0 2024-11-12T06:48:35,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742089_1265 (size=12151) 2024-11-12T06:48:36,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394176077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:36,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:36,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394176087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:36,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:36,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394176087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:36,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:36,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394176087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:36,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:36,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394176087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:36,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:36,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/ad91a8a1ca62452c87d8604123595a6c 2024-11-12T06:48:36,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/944a414fc7674f3ea8a8ea57538c09d4 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/944a414fc7674f3ea8a8ea57538c09d4 2024-11-12T06:48:36,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/944a414fc7674f3ea8a8ea57538c09d4, entries=150, sequenceid=141, filesize=11.9 K 2024-11-12T06:48:36,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/4b897cb89cc44a4ab7b1d137e2947949 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/4b897cb89cc44a4ab7b1d137e2947949 2024-11-12T06:48:36,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/4b897cb89cc44a4ab7b1d137e2947949, entries=150, sequenceid=141, filesize=11.9 K 2024-11-12T06:48:36,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/ad91a8a1ca62452c87d8604123595a6c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ad91a8a1ca62452c87d8604123595a6c 2024-11-12T06:48:36,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ad91a8a1ca62452c87d8604123595a6c, entries=150, sequenceid=141, filesize=11.9 K 2024-11-12T06:48:36,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for abf9890bb5fa9b12fd7d1c33ce62b875 in 865ms, sequenceid=141, compaction requested=false 2024-11-12T06:48:36,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T06:48:36,408 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:36,408 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:48:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-12T06:48:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T06:48:36,410 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:48:36,410 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:48:36,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:48:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T06:48:36,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T06:48:36,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:36,563 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing abf9890bb5fa9b12fd7d1c33ce62b875 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T06:48:36,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=A 2024-11-12T06:48:36,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:36,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=B 2024-11-12T06:48:36,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:36,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=C 2024-11-12T06:48:36,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:36,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/3326e57655f445ebba43c14b1f7cb95e is 50, key is test_row_0/A:col10/1731394115450/Put/seqid=0 2024-11-12T06:48:36,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742090_1266 (size=12151) 2024-11-12T06:48:36,579 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/3326e57655f445ebba43c14b1f7cb95e 2024-11-12T06:48:36,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/510aefbc86c94c5a937e4336751c827a is 50, key is test_row_0/B:col10/1731394115450/Put/seqid=0 2024-11-12T06:48:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:36,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:36,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742091_1267 (size=12151) 2024-11-12T06:48:36,608 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/510aefbc86c94c5a937e4336751c827a 2024-11-12T06:48:36,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394176608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:36,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:36,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394176609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:36,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:36,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:36,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/f28eb0fecaa24a68890c2d362af88b26 is 50, key is test_row_0/C:col10/1731394115450/Put/seqid=0 2024-11-12T06:48:36,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394176612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:36,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394176612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:36,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394176613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:36,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742092_1268 (size=12151) 2024-11-12T06:48:36,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:36,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394176717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:36,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:36,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:36,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394176717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:36,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T06:48:36,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:36,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394176727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:36,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:36,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394176727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:36,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:36,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:36,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394176727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:36,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:36,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:36,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:36,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394176927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:36,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394176928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:36,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:36,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:36,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:36,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394176937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:36,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:36,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394176937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:36,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:36,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:36,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394176937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:36,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:36,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:36,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:37,020 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/f28eb0fecaa24a68890c2d362af88b26 2024-11-12T06:48:37,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/3326e57655f445ebba43c14b1f7cb95e as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/3326e57655f445ebba43c14b1f7cb95e 2024-11-12T06:48:37,028 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/3326e57655f445ebba43c14b1f7cb95e, entries=150, sequenceid=157, filesize=11.9 K 2024-11-12T06:48:37,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/510aefbc86c94c5a937e4336751c827a as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/510aefbc86c94c5a937e4336751c827a 2024-11-12T06:48:37,033 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/510aefbc86c94c5a937e4336751c827a, entries=150, sequenceid=157, filesize=11.9 K 2024-11-12T06:48:37,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/f28eb0fecaa24a68890c2d362af88b26 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/f28eb0fecaa24a68890c2d362af88b26 2024-11-12T06:48:37,037 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/f28eb0fecaa24a68890c2d362af88b26, entries=150, sequenceid=157, filesize=11.9 K 2024-11-12T06:48:37,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T06:48:37,038 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for abf9890bb5fa9b12fd7d1c33ce62b875 in 475ms, sequenceid=157, compaction requested=true 2024-11-12T06:48:37,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:37,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:37,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-12T06:48:37,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-12T06:48:37,040 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-12T06:48:37,040 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 628 msec 2024-11-12T06:48:37,042 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 633 msec 2024-11-12T06:48:37,229 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T06:48:37,229 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-12T06:48:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:37,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing abf9890bb5fa9b12fd7d1c33ce62b875 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T06:48:37,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=A 2024-11-12T06:48:37,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:37,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=B 2024-11-12T06:48:37,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:37,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=C 2024-11-12T06:48:37,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:37,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/b52a77c015364bc5b1223dc3e3cdb994 is 50, key is test_row_0/A:col10/1731394116611/Put/seqid=0 2024-11-12T06:48:37,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742093_1269 (size=14541) 2024-11-12T06:48:37,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394177252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:37,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:37,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394177252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:37,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394177257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:37,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394177257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:37,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394177258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:37,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394177367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:37,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:37,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394177367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:37,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:37,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:37,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394177367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:37,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:37,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:37,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394177367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:37,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:37,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394177367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:37,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T06:48:37,547 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:37,547 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:48:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-12T06:48:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T06:48:37,549 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:48:37,549 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:48:37,550 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:48:37,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:37,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394177577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:37,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:37,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394177577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:37,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:37,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:37,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:37,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394177577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:37,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:37,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:37,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394177577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:37,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:37,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:37,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394177577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:37,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:37,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/b52a77c015364bc5b1223dc3e3cdb994 2024-11-12T06:48:37,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/adc98b76ed044eb289aaefae48a62f82 is 50, key is test_row_0/B:col10/1731394116611/Put/seqid=0 2024-11-12T06:48:37,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T06:48:37,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742094_1270 (size=12151) 2024-11-12T06:48:37,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T06:48:37,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:37,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:37,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:37,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:37,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:37,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:37,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T06:48:37,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:37,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:37,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:37,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:37,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T06:48:37,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394177887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:37,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394177887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394177887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394177887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:37,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:37,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394177897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:38,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T06:48:38,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:38,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:38,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:38,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:38,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:38,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/adc98b76ed044eb289aaefae48a62f82 2024-11-12T06:48:38,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/21dcf50fc4fe4108a4db28361400edc1 is 50, key is test_row_0/C:col10/1731394116611/Put/seqid=0 2024-11-12T06:48:38,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742095_1271 (size=12151) 2024-11-12T06:48:38,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T06:48:38,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:38,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:38,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:38,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:38,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:38,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:38,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T06:48:38,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T06:48:38,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:38,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:38,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:38,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:38,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:38,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394178397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:38,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:38,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394178397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:38,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:38,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394178397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:38,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:38,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394178398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:38,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:38,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394178407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:38,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:38,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:38,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:38,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T06:48:38,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:38,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:38,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:38,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:38,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:38,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:38,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/21dcf50fc4fe4108a4db28361400edc1 2024-11-12T06:48:38,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/b52a77c015364bc5b1223dc3e3cdb994 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/b52a77c015364bc5b1223dc3e3cdb994 2024-11-12T06:48:38,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/b52a77c015364bc5b1223dc3e3cdb994, entries=200, sequenceid=177, filesize=14.2 K 2024-11-12T06:48:38,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/adc98b76ed044eb289aaefae48a62f82 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/adc98b76ed044eb289aaefae48a62f82 2024-11-12T06:48:38,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/adc98b76ed044eb289aaefae48a62f82, entries=150, sequenceid=177, filesize=11.9 K 2024-11-12T06:48:38,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/21dcf50fc4fe4108a4db28361400edc1 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/21dcf50fc4fe4108a4db28361400edc1 2024-11-12T06:48:38,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/21dcf50fc4fe4108a4db28361400edc1, entries=150, sequenceid=177, filesize=11.9 K 2024-11-12T06:48:38,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for abf9890bb5fa9b12fd7d1c33ce62b875 in 1250ms, sequenceid=177, compaction requested=true 2024-11-12T06:48:38,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:38,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:48:38,491 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:48:38,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:38,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:48:38,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:38,491 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:48:38,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:48:38,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:38,492 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:48:38,492 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/B is initiating minor compaction (all files) 2024-11-12T06:48:38,492 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/B in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:38,492 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/6d966b9d979846b9bee9a071ea2c819f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/4b897cb89cc44a4ab7b1d137e2947949, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/510aefbc86c94c5a937e4336751c827a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/adc98b76ed044eb289aaefae48a62f82] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=47.6 K 2024-11-12T06:48:38,492 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51152 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:48:38,492 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/A is initiating minor compaction (all files) 2024-11-12T06:48:38,492 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/A in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:38,492 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/a1ff2ce4c8994712937e6fb343b5e55c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/944a414fc7674f3ea8a8ea57538c09d4, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/3326e57655f445ebba43c14b1f7cb95e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/b52a77c015364bc5b1223dc3e3cdb994] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=50.0 K 2024-11-12T06:48:38,493 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d966b9d979846b9bee9a071ea2c819f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731394114136 2024-11-12T06:48:38,493 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b897cb89cc44a4ab7b1d137e2947949, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731394114787 2024-11-12T06:48:38,493 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1ff2ce4c8994712937e6fb343b5e55c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731394114136 2024-11-12T06:48:38,493 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 510aefbc86c94c5a937e4336751c827a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731394115450 2024-11-12T06:48:38,493 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 944a414fc7674f3ea8a8ea57538c09d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731394114787 2024-11-12T06:48:38,493 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting adc98b76ed044eb289aaefae48a62f82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731394116603 2024-11-12T06:48:38,493 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3326e57655f445ebba43c14b1f7cb95e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731394115450 2024-11-12T06:48:38,494 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting b52a77c015364bc5b1223dc3e3cdb994, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731394116603 2024-11-12T06:48:38,503 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#B#compaction#222 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:38,503 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/fd41aaffef4e452eb03805d419dc1326 is 50, key is test_row_0/B:col10/1731394116611/Put/seqid=0 2024-11-12T06:48:38,510 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#A#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:38,510 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/af5e8a65fd434f0ca1beeffc71a0f08f is 50, key is test_row_0/A:col10/1731394116611/Put/seqid=0 2024-11-12T06:48:38,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742096_1272 (size=12595) 2024-11-12T06:48:38,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742097_1273 (size=12595) 2024-11-12T06:48:38,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T06:48:38,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:38,619 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing abf9890bb5fa9b12fd7d1c33ce62b875 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T06:48:38,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=A 2024-11-12T06:48:38,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:38,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=B 2024-11-12T06:48:38,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:38,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=C 2024-11-12T06:48:38,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:38,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/7abacae8d3ad4b7ea32f896d0bd7130d is 50, key is test_row_0/A:col10/1731394117257/Put/seqid=0 2024-11-12T06:48:38,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742098_1274 (size=12151) 2024-11-12T06:48:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T06:48:38,923 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/fd41aaffef4e452eb03805d419dc1326 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/fd41aaffef4e452eb03805d419dc1326 2024-11-12T06:48:38,927 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/B of abf9890bb5fa9b12fd7d1c33ce62b875 into fd41aaffef4e452eb03805d419dc1326(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:38,927 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:38,927 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/B, priority=12, startTime=1731394118491; duration=0sec 2024-11-12T06:48:38,927 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:38,927 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:B 2024-11-12T06:48:38,928 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:48:38,929 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:48:38,929 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/C is initiating minor compaction (all files) 2024-11-12T06:48:38,929 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/C in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:38,929 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ca273260399d4a9d922ed852527ebcae, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ad91a8a1ca62452c87d8604123595a6c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/f28eb0fecaa24a68890c2d362af88b26, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/21dcf50fc4fe4108a4db28361400edc1] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=47.6 K 2024-11-12T06:48:38,929 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting ca273260399d4a9d922ed852527ebcae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731394114136 2024-11-12T06:48:38,930 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting ad91a8a1ca62452c87d8604123595a6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731394114787 2024-11-12T06:48:38,930 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting f28eb0fecaa24a68890c2d362af88b26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731394115450 2024-11-12T06:48:38,930 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 21dcf50fc4fe4108a4db28361400edc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731394116603 2024-11-12T06:48:38,936 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/af5e8a65fd434f0ca1beeffc71a0f08f as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/af5e8a65fd434f0ca1beeffc71a0f08f 2024-11-12T06:48:38,940 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#C#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:38,941 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/059c5274fb0e4388b09ec654cf6335e5 is 50, key is test_row_0/C:col10/1731394116611/Put/seqid=0 2024-11-12T06:48:38,941 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/A of abf9890bb5fa9b12fd7d1c33ce62b875 into af5e8a65fd434f0ca1beeffc71a0f08f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:38,941 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:38,941 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/A, priority=12, startTime=1731394118491; duration=0sec 2024-11-12T06:48:38,942 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:38,942 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:A 2024-11-12T06:48:38,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742099_1275 (size=12595) 2024-11-12T06:48:38,965 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/059c5274fb0e4388b09ec654cf6335e5 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/059c5274fb0e4388b09ec654cf6335e5 2024-11-12T06:48:38,971 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/C of abf9890bb5fa9b12fd7d1c33ce62b875 into 059c5274fb0e4388b09ec654cf6335e5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:38,971 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:38,971 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/C, priority=12, startTime=1731394118491; duration=0sec 2024-11-12T06:48:38,971 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:38,971 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:C 2024-11-12T06:48:39,027 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/7abacae8d3ad4b7ea32f896d0bd7130d 2024-11-12T06:48:39,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/18903bede2894ca28a74ab8587a0aa51 is 50, key is test_row_0/B:col10/1731394117257/Put/seqid=0 2024-11-12T06:48:39,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742100_1276 (size=12151) 2024-11-12T06:48:39,040 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/18903bede2894ca28a74ab8587a0aa51 2024-11-12T06:48:39,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/6e27a907b6974cb2a5898e7cc8db944e is 50, key is test_row_0/C:col10/1731394117257/Put/seqid=0 2024-11-12T06:48:39,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742101_1277 (size=12151) 2024-11-12T06:48:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:39,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:39,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394179450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394179451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:39,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394179452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394179453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:39,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394179455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:39,466 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/6e27a907b6974cb2a5898e7cc8db944e 2024-11-12T06:48:39,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/7abacae8d3ad4b7ea32f896d0bd7130d as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/7abacae8d3ad4b7ea32f896d0bd7130d 2024-11-12T06:48:39,474 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/7abacae8d3ad4b7ea32f896d0bd7130d, entries=150, sequenceid=190, filesize=11.9 K 2024-11-12T06:48:39,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/18903bede2894ca28a74ab8587a0aa51 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/18903bede2894ca28a74ab8587a0aa51 2024-11-12T06:48:39,479 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/18903bede2894ca28a74ab8587a0aa51, entries=150, sequenceid=190, filesize=11.9 K 2024-11-12T06:48:39,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/6e27a907b6974cb2a5898e7cc8db944e as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/6e27a907b6974cb2a5898e7cc8db944e 2024-11-12T06:48:39,483 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/6e27a907b6974cb2a5898e7cc8db944e, entries=150, sequenceid=190, filesize=11.9 K 2024-11-12T06:48:39,484 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for abf9890bb5fa9b12fd7d1c33ce62b875 in 865ms, sequenceid=190, compaction requested=false 2024-11-12T06:48:39,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:39,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:39,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-12T06:48:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-12T06:48:39,487 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-12T06:48:39,487 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9350 sec 2024-11-12T06:48:39,488 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.9400 sec 2024-11-12T06:48:39,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:39,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing abf9890bb5fa9b12fd7d1c33ce62b875 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T06:48:39,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=A 2024-11-12T06:48:39,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:39,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=B 2024-11-12T06:48:39,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:39,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=C 2024-11-12T06:48:39,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:39,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/4b8c252200694228acd3d4e50a227b48 is 50, key is test_row_0/A:col10/1731394119569/Put/seqid=0 2024-11-12T06:48:39,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394179579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:39,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394179580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:39,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:39,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394179580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:39,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:39,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394179581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394179582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:39,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742102_1278 (size=12151) 2024-11-12T06:48:39,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:39,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394179687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:39,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:39,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T06:48:39,697 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:39,698 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:48:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-12T06:48:39,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394179697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T06:48:39,700 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:48:39,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:39,700 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:48:39,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:48:39,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:39,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394179697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:39,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:39,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394179698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394179698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:39,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T06:48:39,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T06:48:39,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:39,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:39,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:39,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:39,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:39,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394179897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:39,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:39,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394179907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394179907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:39,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394179908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:39,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394179908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:39,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/4b8c252200694228acd3d4e50a227b48 2024-11-12T06:48:39,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/07f1364b394444f693098cbea19a207c is 50, key is test_row_0/B:col10/1731394119569/Put/seqid=0 2024-11-12T06:48:39,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742103_1279 (size=12151) 2024-11-12T06:48:40,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T06:48:40,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:40,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:40,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:40,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:40,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T06:48:40,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T06:48:40,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:40,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:40,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:40,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:40,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:40,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394180207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:40,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394180217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394180217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:40,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:40,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394180217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:40,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:40,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394180218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:40,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:40,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:40,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T06:48:40,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:40,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:40,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:40,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:40,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T06:48:40,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/07f1364b394444f693098cbea19a207c 2024-11-12T06:48:40,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/9036e1ca6c6e40e49e6c831d19707215 is 50, key is test_row_0/C:col10/1731394119569/Put/seqid=0 2024-11-12T06:48:40,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742104_1280 (size=12151) 2024-11-12T06:48:40,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/9036e1ca6c6e40e49e6c831d19707215 2024-11-12T06:48:40,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/4b8c252200694228acd3d4e50a227b48 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/4b8c252200694228acd3d4e50a227b48 2024-11-12T06:48:40,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/4b8c252200694228acd3d4e50a227b48, entries=150, sequenceid=214, filesize=11.9 K 2024-11-12T06:48:40,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/07f1364b394444f693098cbea19a207c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/07f1364b394444f693098cbea19a207c 2024-11-12T06:48:40,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/07f1364b394444f693098cbea19a207c, entries=150, sequenceid=214, filesize=11.9 K 2024-11-12T06:48:40,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/9036e1ca6c6e40e49e6c831d19707215 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/9036e1ca6c6e40e49e6c831d19707215 2024-11-12T06:48:40,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/9036e1ca6c6e40e49e6c831d19707215, entries=150, sequenceid=214, filesize=11.9 K 2024-11-12T06:48:40,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for abf9890bb5fa9b12fd7d1c33ce62b875 in 857ms, sequenceid=214, compaction requested=true 2024-11-12T06:48:40,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:40,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:48:40,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:40,428 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:48:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:40,428 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:48:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:40,428 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:40,428 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:40,428 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/B is initiating minor compaction (all files) 2024-11-12T06:48:40,428 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/A is initiating minor compaction (all files) 2024-11-12T06:48:40,429 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/A in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:40,429 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/B in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:40,429 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/af5e8a65fd434f0ca1beeffc71a0f08f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/7abacae8d3ad4b7ea32f896d0bd7130d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/4b8c252200694228acd3d4e50a227b48] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=36.0 K 2024-11-12T06:48:40,429 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/fd41aaffef4e452eb03805d419dc1326, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/18903bede2894ca28a74ab8587a0aa51, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/07f1364b394444f693098cbea19a207c] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=36.0 K 2024-11-12T06:48:40,429 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting fd41aaffef4e452eb03805d419dc1326, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731394116603 2024-11-12T06:48:40,429 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting af5e8a65fd434f0ca1beeffc71a0f08f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731394116603 2024-11-12T06:48:40,429 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7abacae8d3ad4b7ea32f896d0bd7130d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731394117256 2024-11-12T06:48:40,429 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 18903bede2894ca28a74ab8587a0aa51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731394117256 2024-11-12T06:48:40,430 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b8c252200694228acd3d4e50a227b48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731394119449 2024-11-12T06:48:40,430 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 07f1364b394444f693098cbea19a207c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731394119449 2024-11-12T06:48:40,438 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#A#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:40,438 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#B#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:40,439 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/fc138462d0ff45adb978423a555e47ed is 50, key is test_row_0/B:col10/1731394119569/Put/seqid=0 2024-11-12T06:48:40,439 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/63dea34c8a1547219ee2fd5610aec9b2 is 50, key is test_row_0/A:col10/1731394119569/Put/seqid=0 2024-11-12T06:48:40,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742106_1282 (size=12697) 2024-11-12T06:48:40,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742105_1281 (size=12697) 2024-11-12T06:48:40,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T06:48:40,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:40,463 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing abf9890bb5fa9b12fd7d1c33ce62b875 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T06:48:40,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=A 2024-11-12T06:48:40,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:40,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=B 2024-11-12T06:48:40,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:40,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=C 2024-11-12T06:48:40,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:40,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/0112d2b8b97c48b7b4e1628a0dbf3ebb is 50, key is test_row_0/A:col10/1731394119579/Put/seqid=0 2024-11-12T06:48:40,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742107_1283 (size=12151) 2024-11-12T06:48:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:40,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:40,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:40,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:40,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394180743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:40,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394180742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:40,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731394120737, max timestamp=1731394120747Num uniques 859; 2024-11-12T06:48:40,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T06:48:40,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731394120737, max timestamp=1731394120747Num uniques 859; 2024-11-12T06:48:40,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T06:48:40,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=219, cellSize=10024, totalHeapSize=23016, min timestamp=1731394120737, max timestamp=1731394120747Num uniques 859; 2024-11-12T06:48:40,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=10024, suffix heap size=117064,new segment heap size=23016 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=219 2024-11-12T06:48:40,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:40,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394180797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:40,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:40,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:40,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394180802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:40,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394180802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T06:48:40,859 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/fc138462d0ff45adb978423a555e47ed as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/fc138462d0ff45adb978423a555e47ed 2024-11-12T06:48:40,859 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/63dea34c8a1547219ee2fd5610aec9b2 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/63dea34c8a1547219ee2fd5610aec9b2 2024-11-12T06:48:40,863 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/A of abf9890bb5fa9b12fd7d1c33ce62b875 into 63dea34c8a1547219ee2fd5610aec9b2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:40,863 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/B of abf9890bb5fa9b12fd7d1c33ce62b875 into fc138462d0ff45adb978423a555e47ed(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:40,863 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:40,863 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:40,863 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/B, priority=13, startTime=1731394120427; duration=0sec 2024-11-12T06:48:40,863 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/A, priority=13, startTime=1731394120427; duration=0sec 2024-11-12T06:48:40,863 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:40,864 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:40,864 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:A 2024-11-12T06:48:40,864 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:B 2024-11-12T06:48:40,864 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:40,864 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:40,865 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/C is initiating minor compaction (all files) 2024-11-12T06:48:40,865 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/C in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:40,865 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/059c5274fb0e4388b09ec654cf6335e5, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/6e27a907b6974cb2a5898e7cc8db944e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/9036e1ca6c6e40e49e6c831d19707215] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=36.0 K 2024-11-12T06:48:40,865 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 059c5274fb0e4388b09ec654cf6335e5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731394116603 2024-11-12T06:48:40,865 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e27a907b6974cb2a5898e7cc8db944e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731394117256 2024-11-12T06:48:40,866 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9036e1ca6c6e40e49e6c831d19707215, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731394119449 2024-11-12T06:48:40,871 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#C#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:40,871 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/ebb98ffb615c4361b1f7023f3d20b6a8 is 50, key is test_row_0/C:col10/1731394119569/Put/seqid=0 2024-11-12T06:48:40,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742108_1284 (size=12697) 2024-11-12T06:48:40,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:40,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394180877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:40,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394180877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:40,891 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/0112d2b8b97c48b7b4e1628a0dbf3ebb 2024-11-12T06:48:40,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/d7276369fe854d298d77d7046aad9282 is 50, key is test_row_0/B:col10/1731394119579/Put/seqid=0 2024-11-12T06:48:40,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742109_1285 (size=12151) 2024-11-12T06:48:40,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:40,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394180907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:40,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:40,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:40,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394180917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:40,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:40,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394180917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:41,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:41,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394181087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:41,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:41,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:41,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394181088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:41,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:41,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:41,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394181117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:41,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:41,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:41,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394181127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:41,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:41,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394181127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:41,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:41,278 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/ebb98ffb615c4361b1f7023f3d20b6a8 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ebb98ffb615c4361b1f7023f3d20b6a8 2024-11-12T06:48:41,282 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/C of abf9890bb5fa9b12fd7d1c33ce62b875 into ebb98ffb615c4361b1f7023f3d20b6a8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:41,282 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:41,282 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/C, priority=13, startTime=1731394120428; duration=0sec 2024-11-12T06:48:41,282 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:41,283 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:C 2024-11-12T06:48:41,300 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/d7276369fe854d298d77d7046aad9282 2024-11-12T06:48:41,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/af93507c7e104e12b442fc15456ccc5d is 50, key is test_row_0/C:col10/1731394119579/Put/seqid=0 2024-11-12T06:48:41,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742110_1286 (size=12151) 2024-11-12T06:48:41,325 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/af93507c7e104e12b442fc15456ccc5d 2024-11-12T06:48:41,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/0112d2b8b97c48b7b4e1628a0dbf3ebb as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/0112d2b8b97c48b7b4e1628a0dbf3ebb 2024-11-12T06:48:41,335 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/0112d2b8b97c48b7b4e1628a0dbf3ebb, entries=150, sequenceid=225, filesize=11.9 K 2024-11-12T06:48:41,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/d7276369fe854d298d77d7046aad9282 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/d7276369fe854d298d77d7046aad9282 2024-11-12T06:48:41,339 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/d7276369fe854d298d77d7046aad9282, entries=150, sequenceid=225, filesize=11.9 K 2024-11-12T06:48:41,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/af93507c7e104e12b442fc15456ccc5d as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/af93507c7e104e12b442fc15456ccc5d 2024-11-12T06:48:41,344 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/af93507c7e104e12b442fc15456ccc5d, entries=150, sequenceid=225, filesize=11.9 K 2024-11-12T06:48:41,345 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=132.39 KB/135570 for abf9890bb5fa9b12fd7d1c33ce62b875 in 882ms, sequenceid=225, compaction requested=false 2024-11-12T06:48:41,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:41,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:41,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-12T06:48:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-12T06:48:41,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-12T06:48:41,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6460 sec 2024-11-12T06:48:41,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.6510 sec 2024-11-12T06:48:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:41,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing abf9890bb5fa9b12fd7d1c33ce62b875 3/3 column families, dataSize=139.10 KB heapSize=391.47 KB 2024-11-12T06:48:41,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=A 2024-11-12T06:48:41,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:41,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=B 2024-11-12T06:48:41,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:41,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=C 2024-11-12T06:48:41,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:41,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/d1372216db4140bb8341032fc235053a is 50, key is test_row_0/A:col10/1731394120794/Put/seqid=0 2024-11-12T06:48:41,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742111_1287 (size=14691) 2024-11-12T06:48:41,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:41,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394181420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394181421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:41,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:41,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394181427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:41,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394181437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:41,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:41,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394181437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:41,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394181527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:41,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:41,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394181527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:41,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:41,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394181737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:41,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:41,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394181737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:41,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:41,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:41,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/d1372216db4140bb8341032fc235053a 2024-11-12T06:48:41,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/a1bbc5317df34bc7b189888c19e9bc94 is 50, key is test_row_0/B:col10/1731394120794/Put/seqid=0 2024-11-12T06:48:41,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742112_1288 (size=12251) 2024-11-12T06:48:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T06:48:41,858 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:41,858 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:48:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-12T06:48:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T06:48:41,859 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:48:41,860 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:48:41,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:48:41,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394181937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:41,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:41,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394181947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:41,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394181947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:41,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T06:48:42,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-12T06:48:42,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:42,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:42,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:42,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:42,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:42,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:42,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:42,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394182047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:42,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394182047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:42,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:42,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:42,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:42,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:42,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:42,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:42,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-12T06:48:42,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:42,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:42,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:42,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:42,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T06:48:42,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/a1bbc5317df34bc7b189888c19e9bc94 2024-11-12T06:48:42,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/0697008601694113975a4c03d642a524 is 50, key is test_row_0/C:col10/1731394120794/Put/seqid=0 2024-11-12T06:48:42,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742113_1289 (size=12251) 2024-11-12T06:48:42,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=47.41 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/0697008601694113975a4c03d642a524 2024-11-12T06:48:42,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/d1372216db4140bb8341032fc235053a as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/d1372216db4140bb8341032fc235053a 2024-11-12T06:48:42,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/d1372216db4140bb8341032fc235053a, entries=200, sequenceid=265, filesize=14.3 K 2024-11-12T06:48:42,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/a1bbc5317df34bc7b189888c19e9bc94 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/a1bbc5317df34bc7b189888c19e9bc94 2024-11-12T06:48:42,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/a1bbc5317df34bc7b189888c19e9bc94, entries=150, sequenceid=265, filesize=12.0 K 2024-11-12T06:48:42,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/0697008601694113975a4c03d642a524 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/0697008601694113975a4c03d642a524 2024-11-12T06:48:42,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/0697008601694113975a4c03d642a524, entries=150, sequenceid=265, filesize=12.0 K 2024-11-12T06:48:42,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~145.81 KB/149310, heapSize ~411.34 KB/421216, currentSize=46.96 KB/48090 for abf9890bb5fa9b12fd7d1c33ce62b875 in 845ms, sequenceid=265, compaction requested=true 2024-11-12T06:48:42,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:42,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:48:42,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:42,245 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:42,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:48:42,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:42,245 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:42,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:48:42,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:42,247 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39539 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:42,247 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/A is initiating minor compaction (all files) 2024-11-12T06:48:42,247 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/A in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:42,247 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/63dea34c8a1547219ee2fd5610aec9b2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/0112d2b8b97c48b7b4e1628a0dbf3ebb, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/d1372216db4140bb8341032fc235053a] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=38.6 K 2024-11-12T06:48:42,247 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:42,247 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/B is initiating minor compaction (all files) 2024-11-12T06:48:42,247 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/B in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:42,247 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63dea34c8a1547219ee2fd5610aec9b2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731394119449 2024-11-12T06:48:42,247 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/fc138462d0ff45adb978423a555e47ed, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/d7276369fe854d298d77d7046aad9282, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/a1bbc5317df34bc7b189888c19e9bc94] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=36.2 K 2024-11-12T06:48:42,248 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting fc138462d0ff45adb978423a555e47ed, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731394119449 2024-11-12T06:48:42,248 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0112d2b8b97c48b7b4e1628a0dbf3ebb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731394119579 2024-11-12T06:48:42,248 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting d7276369fe854d298d77d7046aad9282, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731394119579 2024-11-12T06:48:42,248 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1372216db4140bb8341032fc235053a, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731394120794 2024-11-12T06:48:42,248 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting a1bbc5317df34bc7b189888c19e9bc94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731394120794 2024-11-12T06:48:42,255 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#A#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:42,256 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#B#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:42,256 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/11f2c3c79367409f96047a1b60bf8075 is 50, key is test_row_0/A:col10/1731394120794/Put/seqid=0 2024-11-12T06:48:42,256 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/174c4fb27a334bdfb6723a332cfb4908 is 50, key is test_row_0/B:col10/1731394120794/Put/seqid=0 2024-11-12T06:48:42,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742115_1291 (size=12899) 2024-11-12T06:48:42,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742114_1290 (size=12899) 2024-11-12T06:48:42,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-12T06:48:42,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:42,318 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing abf9890bb5fa9b12fd7d1c33ce62b875 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T06:48:42,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=A 2024-11-12T06:48:42,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:42,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=B 2024-11-12T06:48:42,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:42,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=C 2024-11-12T06:48:42,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:42,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/bb52239391d14509be4777c83e1bd7b2 is 50, key is test_row_0/A:col10/1731394121419/Put/seqid=0 2024-11-12T06:48:42,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742116_1292 (size=12301) 2024-11-12T06:48:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T06:48:42,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:42,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:42,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394122588, max timestamp=1731394122614Num uniques 859; 2024-11-12T06:48:42,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:48:42,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394122588, max timestamp=1731394122614Num uniques 859; 2024-11-12T06:48:42,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394122588, max timestamp=1731394122614Num uniques 859; 2024-11-12T06:48:42,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:48:42,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:48:42,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394182658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:42,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:42,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:42,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:42,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394182662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:42,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:42,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:42,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:42,670 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/174c4fb27a334bdfb6723a332cfb4908 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/174c4fb27a334bdfb6723a332cfb4908 2024-11-12T06:48:42,670 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/11f2c3c79367409f96047a1b60bf8075 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/11f2c3c79367409f96047a1b60bf8075 2024-11-12T06:48:42,674 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/B of abf9890bb5fa9b12fd7d1c33ce62b875 into 174c4fb27a334bdfb6723a332cfb4908(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:42,674 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:42,674 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/B, priority=13, startTime=1731394122245; duration=0sec 2024-11-12T06:48:42,674 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:42,674 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:B 2024-11-12T06:48:42,675 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:42,675 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/A of abf9890bb5fa9b12fd7d1c33ce62b875 into 11f2c3c79367409f96047a1b60bf8075(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:42,675 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:42,675 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/A, priority=13, startTime=1731394122245; duration=0sec 2024-11-12T06:48:42,675 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:42,675 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:A 2024-11-12T06:48:42,676 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:42,676 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/C is initiating minor compaction (all files) 2024-11-12T06:48:42,676 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/C in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:42,676 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ebb98ffb615c4361b1f7023f3d20b6a8, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/af93507c7e104e12b442fc15456ccc5d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/0697008601694113975a4c03d642a524] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=36.2 K 2024-11-12T06:48:42,676 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting ebb98ffb615c4361b1f7023f3d20b6a8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731394119449 2024-11-12T06:48:42,676 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting af93507c7e104e12b442fc15456ccc5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731394119579 2024-11-12T06:48:42,677 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 0697008601694113975a4c03d642a524, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731394120794 2024-11-12T06:48:42,683 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#C#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:42,684 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/86e2616b69244993b5aceeb8cdbb17ff is 50, key is test_row_0/C:col10/1731394120794/Put/seqid=0 2024-11-12T06:48:42,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742117_1293 (size=12899) 2024-11-12T06:48:42,725 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/bb52239391d14509be4777c83e1bd7b2 2024-11-12T06:48:42,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/3e288ae833664252974eab07b56cf2a2 is 50, key is test_row_0/B:col10/1731394121419/Put/seqid=0 2024-11-12T06:48:42,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742118_1294 (size=12301) 2024-11-12T06:48:42,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394182767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:42,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394182777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:42,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:42,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:42,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:42,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394182956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:42,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394182957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:42,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394182957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:42,960 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_1' in abf9890bb5fa9b12fd7d1c33ce62b875 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:48:42,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:42,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:42,960 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_1' in abf9890bb5fa9b12fd7d1c33ce62b875 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:48:42,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:42,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:42,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:42,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:42,961 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_2' in abf9890bb5fa9b12fd7d1c33ce62b875 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:48:42,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:42,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:42,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:42,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394182977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:42,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394182987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T06:48:43,092 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/86e2616b69244993b5aceeb8cdbb17ff as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/86e2616b69244993b5aceeb8cdbb17ff 2024-11-12T06:48:43,096 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/C of abf9890bb5fa9b12fd7d1c33ce62b875 into 86e2616b69244993b5aceeb8cdbb17ff(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:43,096 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:43,096 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/C, priority=13, startTime=1731394122245; duration=0sec 2024-11-12T06:48:43,096 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:43,096 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:C 2024-11-12T06:48:43,135 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/3e288ae833664252974eab07b56cf2a2 2024-11-12T06:48:43,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/166093e5b6ad46d2a83ff8bd89d31b63 is 50, key is test_row_0/C:col10/1731394121419/Put/seqid=0 2024-11-12T06:48:43,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742119_1295 (size=12301) 2024-11-12T06:48:43,150 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/166093e5b6ad46d2a83ff8bd89d31b63 2024-11-12T06:48:43,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/bb52239391d14509be4777c83e1bd7b2 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/bb52239391d14509be4777c83e1bd7b2 2024-11-12T06:48:43,159 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/bb52239391d14509be4777c83e1bd7b2, entries=150, sequenceid=275, filesize=12.0 K 2024-11-12T06:48:43,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/3e288ae833664252974eab07b56cf2a2 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/3e288ae833664252974eab07b56cf2a2 2024-11-12T06:48:43,166 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/3e288ae833664252974eab07b56cf2a2, entries=150, sequenceid=275, filesize=12.0 K 2024-11-12T06:48:43,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/166093e5b6ad46d2a83ff8bd89d31b63 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/166093e5b6ad46d2a83ff8bd89d31b63 2024-11-12T06:48:43,171 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/166093e5b6ad46d2a83ff8bd89d31b63, entries=150, sequenceid=275, filesize=12.0 K 2024-11-12T06:48:43,172 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for abf9890bb5fa9b12fd7d1c33ce62b875 in 854ms, sequenceid=275, compaction requested=false 2024-11-12T06:48:43,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:43,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:43,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-12T06:48:43,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-12T06:48:43,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-12T06:48:43,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3130 sec 2024-11-12T06:48:43,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.3180 sec 2024-11-12T06:48:43,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:43,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing abf9890bb5fa9b12fd7d1c33ce62b875 3/3 column families, dataSize=143.13 KB heapSize=410.69 KB 2024-11-12T06:48:43,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=A 2024-11-12T06:48:43,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:48:43,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=B 2024-11-12T06:48:43,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:48:43,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=C 2024-11-12T06:48:43,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:48:43,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/f6f4fbd12c1d42eba4cf3b621612d68a is 50, key is test_row_0/A:col10/1731394122654/Put/seqid=0 2024-11-12T06:48:43,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742120_1296 (size=14741) 2024-11-12T06:48:43,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:43,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394183307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:43,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:43,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:43,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:43,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:43,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394183310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:43,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:43,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:43,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:43,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:43,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394183417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:43,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:43,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:43,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:43,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:43,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394183417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:43,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:43,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:43,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:43,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:43,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394183627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:43,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:43,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394183627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:43,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:43,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:43,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:43,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:43,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:43,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:43,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/f6f4fbd12c1d42eba4cf3b621612d68a 2024-11-12T06:48:43,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/58cf427ac96f49f292d86c71f7dc7656 is 50, key is test_row_0/B:col10/1731394122654/Put/seqid=0 2024-11-12T06:48:43,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742121_1297 (size=12301) 2024-11-12T06:48:43,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394183937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:43,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:43,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:43,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:43,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394183937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T06:48:44,017 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:44,017 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:48:44,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-12T06:48:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T06:48:44,019 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:48:44,020 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:48:44,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:48:44,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/58cf427ac96f49f292d86c71f7dc7656 2024-11-12T06:48:44,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/3924f42139984dcca463ac21f68075ad is 50, key is test_row_0/C:col10/1731394122654/Put/seqid=0 2024-11-12T06:48:44,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742122_1298 (size=12301) 2024-11-12T06:48:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T06:48:44,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T06:48:44,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:44,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:44,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:44,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:44,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:44,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:44,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T06:48:44,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:44,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:44,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:44,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:44,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T06:48:44,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394184447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:44,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394184447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:44,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T06:48:44,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:44,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:44,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:44,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:44,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:44,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/3924f42139984dcca463ac21f68075ad 2024-11-12T06:48:44,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/f6f4fbd12c1d42eba4cf3b621612d68a as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/f6f4fbd12c1d42eba4cf3b621612d68a 2024-11-12T06:48:44,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/f6f4fbd12c1d42eba4cf3b621612d68a, entries=200, sequenceid=316, filesize=14.4 K 2024-11-12T06:48:44,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/58cf427ac96f49f292d86c71f7dc7656 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/58cf427ac96f49f292d86c71f7dc7656 2024-11-12T06:48:44,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/58cf427ac96f49f292d86c71f7dc7656, entries=150, sequenceid=316, filesize=12.0 K 2024-11-12T06:48:44,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/3924f42139984dcca463ac21f68075ad as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/3924f42139984dcca463ac21f68075ad 2024-11-12T06:48:44,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/3924f42139984dcca463ac21f68075ad, entries=150, sequenceid=316, filesize=12.0 K 2024-11-12T06:48:44,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~143.13 KB/146560, heapSize ~410.64 KB/420496, currentSize=40.25 KB/41220 for abf9890bb5fa9b12fd7d1c33ce62b875 in 1246ms, sequenceid=316, compaction requested=true 2024-11-12T06:48:44,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:44,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:48:44,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:44,537 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:44,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:48:44,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:44,537 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:44,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:48:44,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:44,538 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39941 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:44,538 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/A is initiating minor compaction (all files) 2024-11-12T06:48:44,538 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/A in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:44,538 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/11f2c3c79367409f96047a1b60bf8075, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/bb52239391d14509be4777c83e1bd7b2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/f6f4fbd12c1d42eba4cf3b621612d68a] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=39.0 K 2024-11-12T06:48:44,538 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:44,538 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/B is initiating minor compaction (all files) 2024-11-12T06:48:44,539 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/B in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:44,539 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/174c4fb27a334bdfb6723a332cfb4908, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/3e288ae833664252974eab07b56cf2a2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/58cf427ac96f49f292d86c71f7dc7656] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=36.6 K 2024-11-12T06:48:44,539 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 174c4fb27a334bdfb6723a332cfb4908, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731394120794 2024-11-12T06:48:44,539 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11f2c3c79367409f96047a1b60bf8075, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731394120794 2024-11-12T06:48:44,539 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb52239391d14509be4777c83e1bd7b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731394121412 2024-11-12T06:48:44,539 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e288ae833664252974eab07b56cf2a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731394121412 2024-11-12T06:48:44,539 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6f4fbd12c1d42eba4cf3b621612d68a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731394122649 2024-11-12T06:48:44,539 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 58cf427ac96f49f292d86c71f7dc7656, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731394122649 2024-11-12T06:48:44,546 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#B#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:44,546 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/009cef0895084fd9aca58f603a3bb66b is 50, key is test_row_0/B:col10/1731394122654/Put/seqid=0 2024-11-12T06:48:44,548 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#A#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:44,549 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/2a91978d8a124110a7b539c3b615bcc2 is 50, key is test_row_0/A:col10/1731394122654/Put/seqid=0 2024-11-12T06:48:44,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742124_1300 (size=13051) 2024-11-12T06:48:44,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742123_1299 (size=13051) 2024-11-12T06:48:44,567 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/2a91978d8a124110a7b539c3b615bcc2 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/2a91978d8a124110a7b539c3b615bcc2 2024-11-12T06:48:44,570 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/009cef0895084fd9aca58f603a3bb66b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/009cef0895084fd9aca58f603a3bb66b 2024-11-12T06:48:44,571 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/A of abf9890bb5fa9b12fd7d1c33ce62b875 into 2a91978d8a124110a7b539c3b615bcc2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:44,571 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:44,571 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/A, priority=13, startTime=1731394124537; duration=0sec 2024-11-12T06:48:44,571 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:44,571 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:A 2024-11-12T06:48:44,571 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:44,573 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:44,573 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/C is initiating minor compaction (all files) 2024-11-12T06:48:44,573 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/C in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:44,573 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/86e2616b69244993b5aceeb8cdbb17ff, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/166093e5b6ad46d2a83ff8bd89d31b63, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/3924f42139984dcca463ac21f68075ad] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=36.6 K 2024-11-12T06:48:44,574 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86e2616b69244993b5aceeb8cdbb17ff, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731394120794 2024-11-12T06:48:44,574 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 166093e5b6ad46d2a83ff8bd89d31b63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731394121412 2024-11-12T06:48:44,575 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3924f42139984dcca463ac21f68075ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731394122649 2024-11-12T06:48:44,576 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/B of abf9890bb5fa9b12fd7d1c33ce62b875 into 009cef0895084fd9aca58f603a3bb66b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:44,576 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:44,576 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/B, priority=13, startTime=1731394124537; duration=0sec 2024-11-12T06:48:44,576 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:44,576 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:B 2024-11-12T06:48:44,582 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#C#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:44,583 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/35565ebe50c248909f9d4e891d8bdaf0 is 50, key is test_row_0/C:col10/1731394122654/Put/seqid=0 2024-11-12T06:48:44,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742125_1301 (size=13051) 2024-11-12T06:48:44,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T06:48:44,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:44,630 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing abf9890bb5fa9b12fd7d1c33ce62b875 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-12T06:48:44,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=A 2024-11-12T06:48:44,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:44,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=B 2024-11-12T06:48:44,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:44,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=C 2024-11-12T06:48:44,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:44,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/4c8d5e0fb6814fb38ebe35a917cda3a9 is 50, key is test_row_0/A:col10/1731394123301/Put/seqid=0 2024-11-12T06:48:44,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742126_1302 (size=12301) 2024-11-12T06:48:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T06:48:44,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:44,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:44,993 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/35565ebe50c248909f9d4e891d8bdaf0 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/35565ebe50c248909f9d4e891d8bdaf0 2024-11-12T06:48:44,997 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/C of abf9890bb5fa9b12fd7d1c33ce62b875 into 35565ebe50c248909f9d4e891d8bdaf0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:44,997 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:44,997 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/C, priority=13, startTime=1731394124537; duration=0sec 2024-11-12T06:48:44,997 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:44,997 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:C 2024-11-12T06:48:45,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394124986, max timestamp=1731394125003Num uniques 859; 2024-11-12T06:48:45,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394124986, max timestamp=1731394125003Num uniques 859; 2024-11-12T06:48:45,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394124986, max timestamp=1731394125003Num uniques 859; 2024-11-12T06:48:45,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:48:45,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:48:45,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:48:45,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:45,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394185031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:45,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:45,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:45,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394185032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:45,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:45,039 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/4c8d5e0fb6814fb38ebe35a917cda3a9 2024-11-12T06:48:45,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:45,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394185036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:45,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:45,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/ad55e00231874dddab5c3d66cac0515a is 50, key is test_row_0/B:col10/1731394123301/Put/seqid=0 2024-11-12T06:48:45,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742127_1303 (size=12301) 2024-11-12T06:48:45,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:45,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394185147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:45,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:45,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394185147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:45,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:45,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:45,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394185147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:45,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:45,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T06:48:45,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394185357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:45,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:45,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394185357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:45,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:45,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394185357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:45,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:45,449 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/ad55e00231874dddab5c3d66cac0515a 2024-11-12T06:48:45,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/1da3750e19a447f885bd7d8d1c2ef75f is 50, key is test_row_0/C:col10/1731394123301/Put/seqid=0 2024-11-12T06:48:45,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394185457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:45,458 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_0' in abf9890bb5fa9b12fd7d1c33ce62b875 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:48:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:45,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742128_1304 (size=12301) 2024-11-12T06:48:45,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394185467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:45,470 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_1' in abf9890bb5fa9b12fd7d1c33ce62b875 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:48:45,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:45,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394185667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:45,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394185667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:45,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394185667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:45,860 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/1da3750e19a447f885bd7d8d1c2ef75f 2024-11-12T06:48:45,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/4c8d5e0fb6814fb38ebe35a917cda3a9 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/4c8d5e0fb6814fb38ebe35a917cda3a9 2024-11-12T06:48:45,867 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/4c8d5e0fb6814fb38ebe35a917cda3a9, entries=150, sequenceid=327, filesize=12.0 K 2024-11-12T06:48:45,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/ad55e00231874dddab5c3d66cac0515a as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/ad55e00231874dddab5c3d66cac0515a 2024-11-12T06:48:45,871 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/ad55e00231874dddab5c3d66cac0515a, entries=150, sequenceid=327, filesize=12.0 K 2024-11-12T06:48:45,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/1da3750e19a447f885bd7d8d1c2ef75f as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/1da3750e19a447f885bd7d8d1c2ef75f 2024-11-12T06:48:45,875 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/1da3750e19a447f885bd7d8d1c2ef75f, entries=150, sequenceid=327, filesize=12.0 K 2024-11-12T06:48:45,876 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for abf9890bb5fa9b12fd7d1c33ce62b875 in 1246ms, sequenceid=327, compaction requested=false 2024-11-12T06:48:45,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:45,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:45,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-12T06:48:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-12T06:48:45,879 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-12T06:48:45,879 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8570 sec 2024-11-12T06:48:45,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.8610 sec 2024-11-12T06:48:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T06:48:46,168 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:46,168 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:48:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-12T06:48:46,169 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:48:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T06:48:46,170 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:48:46,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:48:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:46,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing abf9890bb5fa9b12fd7d1c33ce62b875 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-12T06:48:46,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731394125028, max timestamp=1731394126179Num uniques 1068; 2024-11-12T06:48:46,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-12T06:48:46,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731394125028, max timestamp=1731394126179Num uniques 1068; 2024-11-12T06:48:46,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-12T06:48:46,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731394125028, max timestamp=1731394126179Num uniques 1068; 2024-11-12T06:48:46,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-12T06:48:46,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=A 2024-11-12T06:48:46,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:46,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=B 2024-11-12T06:48:46,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:46,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=C 2024-11-12T06:48:46,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:46,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/5b287c969d66439ab438c50664f64a71 is 50, key is test_row_0/A:col10/1731394125034/Put/seqid=0 2024-11-12T06:48:46,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742129_1305 (size=14741) 2024-11-12T06:48:46,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394126200, max timestamp=1731394126217Num uniques 859; 2024-11-12T06:48:46,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:48:46,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394126200, max timestamp=1731394126217Num uniques 859; 2024-11-12T06:48:46,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:48:46,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731394126200, max timestamp=1731394126217Num uniques 859; 2024-11-12T06:48:46,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:48:46,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9968, totalHeapSize=22912, min timestamp=1731394126233, max timestamp=1731394126258Num uniques 1068; 2024-11-12T06:48:46,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9968, totalHeapSize=22912, min timestamp=1731394126233, max timestamp=1731394126258Num uniques 1068; 2024-11-12T06:48:46,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48898, new segment data size=9968, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T06:48:46,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48898, new segment data size=9968, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T06:48:46,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9968, totalHeapSize=22912, min timestamp=1731394126233, max timestamp=1731394126258Num uniques 1068; 2024-11-12T06:48:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T06:48:46,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48898, new segment data size=9968, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T06:48:46,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731394126295, max timestamp=1731394126307Num uniques 1077; 2024-11-12T06:48:46,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731394126295, max timestamp=1731394126307Num uniques 1077; 2024-11-12T06:48:46,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-12T06:48:46,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-12T06:48:46,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731394126295, max timestamp=1731394126307Num uniques 1077; 2024-11-12T06:48:46,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-12T06:48:46,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T06:48:46,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:46,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:46,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:46,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:46,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:46,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731394126358, max timestamp=1731394126374Num uniques 1086; 2024-11-12T06:48:46,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-12T06:48:46,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731394126358, max timestamp=1731394126374Num uniques 1086; 2024-11-12T06:48:46,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-12T06:48:46,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731394126358, max timestamp=1731394126374Num uniques 1086; 2024-11-12T06:48:46,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-12T06:48:46,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731394126400, max timestamp=1731394126417Num uniques 1145; 2024-11-12T06:48:46,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731394126400, max timestamp=1731394126417Num uniques 1145; 2024-11-12T06:48:46,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-12T06:48:46,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-12T06:48:46,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731394126400, max timestamp=1731394126417Num uniques 1145; 2024-11-12T06:48:46,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-12T06:48:46,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394186450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:46,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:46,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:46,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:46,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731394126445, max timestamp=1731394126454Num uniques 1104; 2024-11-12T06:48:46,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-12T06:48:46,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731394126445, max timestamp=1731394126454Num uniques 1104; 2024-11-12T06:48:46,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731394126445, max timestamp=1731394126454Num uniques 1104; 2024-11-12T06:48:46,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-12T06:48:46,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-12T06:48:46,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T06:48:46,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:46,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:46,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:46,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:46,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T06:48:46,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731394126495, max timestamp=1731394126507Num uniques 1063; 2024-11-12T06:48:46,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-12T06:48:46,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731394126495, max timestamp=1731394126507Num uniques 1063; 2024-11-12T06:48:46,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-12T06:48:46,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731394126495, max timestamp=1731394126507Num uniques 1063; 2024-11-12T06:48:46,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-12T06:48:46,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731394126537, max timestamp=1731394126552Num uniques 1022; 2024-11-12T06:48:46,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-12T06:48:46,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731394126537, max timestamp=1731394126552Num uniques 1022; 2024-11-12T06:48:46,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-12T06:48:46,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731394126537, max timestamp=1731394126552Num uniques 1022; 2024-11-12T06:48:46,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-12T06:48:46,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731394126581, max timestamp=1731394126595Num uniques 1031; 2024-11-12T06:48:46,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=12866, suffix heap size=135192,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=281 2024-11-12T06:48:46,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731394126581, max timestamp=1731394126595Num uniques 1031; 2024-11-12T06:48:46,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=12866, suffix heap size=135192,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=281 2024-11-12T06:48:46,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731394126581, max timestamp=1731394126595Num uniques 1031; 2024-11-12T06:48:46,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=12866, suffix heap size=135192,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=281 2024-11-12T06:48:46,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/5b287c969d66439ab438c50664f64a71 2024-11-12T06:48:46,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/0a465946a1f64cf59a32cfc43b3aeb14 is 50, key is test_row_0/B:col10/1731394125034/Put/seqid=0 2024-11-12T06:48:46,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742130_1306 (size=12301) 2024-11-12T06:48:46,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T06:48:46,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:46,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:46,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:46,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:46,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:46,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731394126620, max timestamp=1731394126637Num uniques 1140; 2024-11-12T06:48:46,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731394126620, max timestamp=1731394126637Num uniques 1140; 2024-11-12T06:48:46,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=10990, suffix heap size=146528,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=240 2024-11-12T06:48:46,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=10990, suffix heap size=146528,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=240 2024-11-12T06:48:46,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731394126620, max timestamp=1731394126637Num uniques 1140; 2024-11-12T06:48:46,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=10990, suffix heap size=146528,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=240 2024-11-12T06:48:46,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731394126664, max timestamp=1731394126672Num uniques 1099; 2024-11-12T06:48:46,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=13694, suffix heap size=142264,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=299 2024-11-12T06:48:46,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731394126664, max timestamp=1731394126672Num uniques 1099; 2024-11-12T06:48:46,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=13694, suffix heap size=142264,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=299 2024-11-12T06:48:46,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731394126664, max timestamp=1731394126672Num uniques 1099; 2024-11-12T06:48:46,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=13694, suffix heap size=142264,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=299 2024-11-12T06:48:46,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:46,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394186706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:46,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:46,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:46,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:46,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394186709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:46,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394186710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:46,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T06:48:46,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:46,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:46,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:46,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:46,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T06:48:46,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:46,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394186817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:46,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:46,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394186827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:46,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:46,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394186827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:46,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T06:48:46,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:46,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:46,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:46,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:46,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:47,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/0a465946a1f64cf59a32cfc43b3aeb14 2024-11-12T06:48:47,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/2bdc98965d5741449ffb52769204d0a6 is 50, key is test_row_0/C:col10/1731394125034/Put/seqid=0 2024-11-12T06:48:47,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:47,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394187027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:47,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742131_1307 (size=12301) 2024-11-12T06:48:47,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/2bdc98965d5741449ffb52769204d0a6 2024-11-12T06:48:47,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:47,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394187037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:47,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/5b287c969d66439ab438c50664f64a71 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/5b287c969d66439ab438c50664f64a71 2024-11-12T06:48:47,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394187037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:47,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/5b287c969d66439ab438c50664f64a71, entries=200, sequenceid=367, filesize=14.4 K 2024-11-12T06:48:47,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/0a465946a1f64cf59a32cfc43b3aeb14 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/0a465946a1f64cf59a32cfc43b3aeb14 2024-11-12T06:48:47,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/0a465946a1f64cf59a32cfc43b3aeb14, entries=150, sequenceid=367, filesize=12.0 K 2024-11-12T06:48:47,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/2bdc98965d5741449ffb52769204d0a6 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/2bdc98965d5741449ffb52769204d0a6 2024-11-12T06:48:47,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/2bdc98965d5741449ffb52769204d0a6, entries=150, sequenceid=367, filesize=12.0 K 2024-11-12T06:48:47,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=154.31 KB/158010 for abf9890bb5fa9b12fd7d1c33ce62b875 in 869ms, sequenceid=367, compaction requested=true 2024-11-12T06:48:47,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:47,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:48:47,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:47,050 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:47,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:48:47,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:47,051 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:47,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:48:47,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:47,051 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:47,051 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:47,051 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/A is initiating minor compaction (all files) 2024-11-12T06:48:47,051 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/B is initiating minor compaction (all files) 2024-11-12T06:48:47,051 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/B in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:47,051 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/A in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:47,051 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/2a91978d8a124110a7b539c3b615bcc2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/4c8d5e0fb6814fb38ebe35a917cda3a9, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/5b287c969d66439ab438c50664f64a71] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=39.2 K 2024-11-12T06:48:47,051 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/009cef0895084fd9aca58f603a3bb66b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/ad55e00231874dddab5c3d66cac0515a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/0a465946a1f64cf59a32cfc43b3aeb14] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=36.8 K 2024-11-12T06:48:47,052 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 009cef0895084fd9aca58f603a3bb66b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731394122649 2024-11-12T06:48:47,052 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a91978d8a124110a7b539c3b615bcc2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731394122649 2024-11-12T06:48:47,052 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c8d5e0fb6814fb38ebe35a917cda3a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731394123301 2024-11-12T06:48:47,052 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting ad55e00231874dddab5c3d66cac0515a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731394123301 2024-11-12T06:48:47,052 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b287c969d66439ab438c50664f64a71, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731394125028 2024-11-12T06:48:47,052 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a465946a1f64cf59a32cfc43b3aeb14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731394125028 2024-11-12T06:48:47,058 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#B#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:47,058 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:47,058 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/315a0bcb16494036af8d7441e9cfaeb9 is 50, key is test_row_0/B:col10/1731394125034/Put/seqid=0 2024-11-12T06:48:47,059 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/7bb4b1e7a68f4aa4878fb63b1ee617d2 is 50, key is test_row_0/A:col10/1731394125034/Put/seqid=0 2024-11-12T06:48:47,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742133_1309 (size=13153) 2024-11-12T06:48:47,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742132_1308 (size=13153) 2024-11-12T06:48:47,071 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/7bb4b1e7a68f4aa4878fb63b1ee617d2 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/7bb4b1e7a68f4aa4878fb63b1ee617d2 2024-11-12T06:48:47,075 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/A of abf9890bb5fa9b12fd7d1c33ce62b875 into 7bb4b1e7a68f4aa4878fb63b1ee617d2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:47,075 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:47,075 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/A, priority=13, startTime=1731394127050; duration=0sec 2024-11-12T06:48:47,075 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:47,075 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:A 2024-11-12T06:48:47,075 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:47,075 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:47,075 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/C is initiating minor compaction (all files) 2024-11-12T06:48:47,076 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/C in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:47,076 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/35565ebe50c248909f9d4e891d8bdaf0, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/1da3750e19a447f885bd7d8d1c2ef75f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/2bdc98965d5741449ffb52769204d0a6] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=36.8 K 2024-11-12T06:48:47,076 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35565ebe50c248909f9d4e891d8bdaf0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731394122649 2024-11-12T06:48:47,076 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1da3750e19a447f885bd7d8d1c2ef75f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731394123301 2024-11-12T06:48:47,076 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bdc98965d5741449ffb52769204d0a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731394125028 2024-11-12T06:48:47,082 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#C#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:47,083 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/ae0a3a82fa4c447fbd84d48e8386d83d is 50, key is test_row_0/C:col10/1731394125034/Put/seqid=0 2024-11-12T06:48:47,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742134_1310 (size=13153) 2024-11-12T06:48:47,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T06:48:47,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:47,089 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing abf9890bb5fa9b12fd7d1c33ce62b875 3/3 column families, dataSize=154.31 KB heapSize=431.63 KB 2024-11-12T06:48:47,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=A 2024-11-12T06:48:47,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:47,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=B 2024-11-12T06:48:47,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:47,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=C 2024-11-12T06:48:47,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:47,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/76e5a833de00408880b3aad5414f810c is 50, key is test_row_0/A:col10/1731394126700/Put/seqid=0 2024-11-12T06:48:47,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742135_1311 (size=12301) 2024-11-12T06:48:47,099 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/76e5a833de00408880b3aad5414f810c 2024-11-12T06:48:47,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/86c199f7dca341d9a8a0a23c93e46ccb is 50, key is test_row_0/B:col10/1731394126700/Put/seqid=0 2024-11-12T06:48:47,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742136_1312 (size=12301) 2024-11-12T06:48:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T06:48:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:47,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:47,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394187350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:47,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394187351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394187351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:47,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:47,471 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/315a0bcb16494036af8d7441e9cfaeb9 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/315a0bcb16494036af8d7441e9cfaeb9 2024-11-12T06:48:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394187467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:47,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394187467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:47,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394187467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:47,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:47,477 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/B of abf9890bb5fa9b12fd7d1c33ce62b875 into 315a0bcb16494036af8d7441e9cfaeb9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:47,477 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:47,477 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/B, priority=13, startTime=1731394127050; duration=0sec 2024-11-12T06:48:47,477 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:47,477 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:B 2024-11-12T06:48:47,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37258 deadline: 1731394187477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:47,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37232 deadline: 1731394187477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:47,481 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_1' in abf9890bb5fa9b12fd7d1c33ce62b875 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:48:47,481 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_0' in abf9890bb5fa9b12fd7d1c33ce62b875 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:48:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:47,490 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/ae0a3a82fa4c447fbd84d48e8386d83d as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ae0a3a82fa4c447fbd84d48e8386d83d 2024-11-12T06:48:47,495 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/C of abf9890bb5fa9b12fd7d1c33ce62b875 into ae0a3a82fa4c447fbd84d48e8386d83d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:47,495 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:47,495 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/C, priority=13, startTime=1731394127051; duration=0sec 2024-11-12T06:48:47,495 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:47,495 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:C 2024-11-12T06:48:47,513 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/86c199f7dca341d9a8a0a23c93e46ccb 2024-11-12T06:48:47,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/5a7ec4624817417398c30dcda6d997f4 is 50, key is test_row_0/C:col10/1731394126700/Put/seqid=0 2024-11-12T06:48:47,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742137_1313 (size=12301) 2024-11-12T06:48:47,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394187677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:47,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394187677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:47,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394187687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:47,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:47,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:47,924 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/5a7ec4624817417398c30dcda6d997f4 2024-11-12T06:48:47,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/76e5a833de00408880b3aad5414f810c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/76e5a833de00408880b3aad5414f810c 2024-11-12T06:48:47,930 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/76e5a833de00408880b3aad5414f810c, entries=150, sequenceid=577, filesize=12.0 K 2024-11-12T06:48:47,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/86c199f7dca341d9a8a0a23c93e46ccb as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/86c199f7dca341d9a8a0a23c93e46ccb 2024-11-12T06:48:47,938 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/86c199f7dca341d9a8a0a23c93e46ccb, entries=150, sequenceid=577, filesize=12.0 K 2024-11-12T06:48:47,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/5a7ec4624817417398c30dcda6d997f4 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/5a7ec4624817417398c30dcda6d997f4 2024-11-12T06:48:47,942 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/5a7ec4624817417398c30dcda6d997f4, entries=150, sequenceid=577, filesize=12.0 K 2024-11-12T06:48:47,943 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~431.58 KB/441936, currentSize=33.54 KB/34350 for abf9890bb5fa9b12fd7d1c33ce62b875 in 854ms, sequenceid=577, compaction requested=false 2024-11-12T06:48:47,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:47,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:47,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-12T06:48:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-12T06:48:47,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-12T06:48:47,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7750 sec 2024-11-12T06:48:47,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.7790 sec 2024-11-12T06:48:47,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:47,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing abf9890bb5fa9b12fd7d1c33ce62b875 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T06:48:47,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=A 2024-11-12T06:48:47,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:47,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=B 2024-11-12T06:48:47,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:47,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=C 2024-11-12T06:48:47,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:47,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/f3e5f5dfe0c348c4a48333dddde7a9a6 is 50, key is test_row_1/A:col10/1731394127988/Put/seqid=0 2024-11-12T06:48:48,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742138_1314 (size=12297) 2024-11-12T06:48:48,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394128010, max timestamp=1731394128020Num uniques 859; 2024-11-12T06:48:48,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394128010, max timestamp=1731394128020Num uniques 859; 2024-11-12T06:48:48,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:48:48,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:48:48,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394128010, max timestamp=1731394128020Num uniques 859; 2024-11-12T06:48:48,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:48:48,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:48,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394188044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:48,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:48,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394188046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:48,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:48,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:48,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394188049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:48,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394188157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:48,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394188157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:48,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394188157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:48,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T06:48:48,317 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:48,318 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:48:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-12T06:48:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T06:48:48,319 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:48:48,319 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:48:48,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:48:48,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394188367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:48,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:48,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394188367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:48,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:48,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:48,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:48,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394188367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:48,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:48,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=589 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/f3e5f5dfe0c348c4a48333dddde7a9a6 2024-11-12T06:48:48,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/146607a57e8d4abdb3f4afa359370373 is 50, key is test_row_1/B:col10/1731394127988/Put/seqid=0 2024-11-12T06:48:48,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742139_1315 (size=9857) 2024-11-12T06:48:48,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=589 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/146607a57e8d4abdb3f4afa359370373 2024-11-12T06:48:48,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/c1deceb3da5544cc86a147d946964fb1 is 50, key is test_row_1/C:col10/1731394127988/Put/seqid=0 2024-11-12T06:48:48,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742140_1316 (size=9857) 2024-11-12T06:48:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T06:48:48,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-12T06:48:48,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:48,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:48,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:48,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:48,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:48,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-12T06:48:48,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:48,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:48,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:48,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:48,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:48,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T06:48:48,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37278 deadline: 1731394188677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:48,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:48,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:48,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37284 deadline: 1731394188678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:48,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:48,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:48,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37280 deadline: 1731394188678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:48,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:48,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:48:48,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-12T06:48:48,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:48,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:48,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:48,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:48,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:48,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:48,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=589 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/c1deceb3da5544cc86a147d946964fb1 2024-11-12T06:48:48,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/f3e5f5dfe0c348c4a48333dddde7a9a6 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/f3e5f5dfe0c348c4a48333dddde7a9a6 2024-11-12T06:48:48,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/f3e5f5dfe0c348c4a48333dddde7a9a6, entries=150, sequenceid=589, filesize=12.0 K 2024-11-12T06:48:48,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/146607a57e8d4abdb3f4afa359370373 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/146607a57e8d4abdb3f4afa359370373 2024-11-12T06:48:48,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/146607a57e8d4abdb3f4afa359370373, entries=100, sequenceid=589, filesize=9.6 K 2024-11-12T06:48:48,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/c1deceb3da5544cc86a147d946964fb1 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/c1deceb3da5544cc86a147d946964fb1 2024-11-12T06:48:48,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/c1deceb3da5544cc86a147d946964fb1, entries=100, sequenceid=589, filesize=9.6 K 2024-11-12T06:48:48,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for abf9890bb5fa9b12fd7d1c33ce62b875 in 847ms, sequenceid=589, compaction requested=true 2024-11-12T06:48:48,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:48,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:48:48,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:48,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:48:48,836 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:48,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:48,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abf9890bb5fa9b12fd7d1c33ce62b875:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:48:48,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:48,836 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:48,837 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:48,837 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37751 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:48,837 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/B is initiating minor compaction (all files) 2024-11-12T06:48:48,837 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/A is initiating minor compaction (all files) 2024-11-12T06:48:48,837 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/B in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:48,837 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/A in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:48,837 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/315a0bcb16494036af8d7441e9cfaeb9, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/86c199f7dca341d9a8a0a23c93e46ccb, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/146607a57e8d4abdb3f4afa359370373] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=34.5 K 2024-11-12T06:48:48,837 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/7bb4b1e7a68f4aa4878fb63b1ee617d2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/76e5a833de00408880b3aad5414f810c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/f3e5f5dfe0c348c4a48333dddde7a9a6] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=36.9 K 2024-11-12T06:48:48,837 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 315a0bcb16494036af8d7441e9cfaeb9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731394125028 2024-11-12T06:48:48,837 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bb4b1e7a68f4aa4878fb63b1ee617d2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731394125028 2024-11-12T06:48:48,837 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76e5a833de00408880b3aad5414f810c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1731394126700 2024-11-12T06:48:48,837 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 86c199f7dca341d9a8a0a23c93e46ccb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1731394126700 2024-11-12T06:48:48,838 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3e5f5dfe0c348c4a48333dddde7a9a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=589, earliestPutTs=1731394127350 2024-11-12T06:48:48,838 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 146607a57e8d4abdb3f4afa359370373, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=589, earliestPutTs=1731394127988 2024-11-12T06:48:48,843 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#A#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:48,843 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/861859c95bf74d28ab8f1b7b1f994ccf is 50, key is test_row_0/A:col10/1731394126700/Put/seqid=0 2024-11-12T06:48:48,843 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#B#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:48,844 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/5ceebd8f87c6425385bb2bb23e5d54ee is 50, key is test_row_0/B:col10/1731394126700/Put/seqid=0 2024-11-12T06:48:48,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742142_1318 (size=13255) 2024-11-12T06:48:48,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742141_1317 (size=13255) 2024-11-12T06:48:48,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-12T06:48:48,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:48,930 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing abf9890bb5fa9b12fd7d1c33ce62b875 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-12T06:48:48,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=A 2024-11-12T06:48:48,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:48,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=B 2024-11-12T06:48:48,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:48,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=C 2024-11-12T06:48:48,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:48,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/4451198e05ac4abbbcb3002e2b55bd67 is 50, key is test_row_0/A:col10/1731394128048/Put/seqid=0 2024-11-12T06:48:48,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742143_1319 (size=12301) 2024-11-12T06:48:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T06:48:48,947 INFO [Thread-1103 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1103. 2024-11-12T06:48:48,947 DEBUG [Thread-1103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:48:48,947 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:48,947 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:48,947 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:48:48,948 INFO [Thread-1099 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1099. 2024-11-12T06:48:48,948 DEBUG [Thread-1099 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:48:48,948 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:48,949 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:48,949 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:48:48,950 INFO [Thread-1101 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1101. 2024-11-12T06:48:48,951 DEBUG [Thread-1101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:48:48,951 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:48,951 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:48,951 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:48:48,951 INFO [Thread-1095 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1095. 2024-11-12T06:48:48,951 DEBUG [Thread-1095 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:48:48,951 DEBUG [Thread-1095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:48,951 DEBUG [Thread-1095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:48,951 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:48:48,952 INFO [Thread-1097 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1097. 2024-11-12T06:48:48,953 DEBUG [Thread-1097 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:48:48,953 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:48,953 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:48,953 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:48:49,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:49,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. as already flushing 2024-11-12T06:48:49,192 INFO [Thread-1092 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1092. 2024-11-12T06:48:49,192 INFO [Thread-1084 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1084. 2024-11-12T06:48:49,192 DEBUG [Thread-1084 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:48:49,192 DEBUG [Thread-1092 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:48:49,192 DEBUG [Thread-1084 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:49,192 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:49,192 INFO [Thread-1090 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1090. 2024-11-12T06:48:49,192 DEBUG [Thread-1084 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:49,192 DEBUG [Thread-1090 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:48:49,192 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:49,192 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:48:49,192 DEBUG [Thread-1090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:49,192 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:48:49,192 DEBUG [Thread-1090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:49,192 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:48:49,259 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/861859c95bf74d28ab8f1b7b1f994ccf as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/861859c95bf74d28ab8f1b7b1f994ccf 2024-11-12T06:48:49,259 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/5ceebd8f87c6425385bb2bb23e5d54ee as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/5ceebd8f87c6425385bb2bb23e5d54ee 2024-11-12T06:48:49,265 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/A of abf9890bb5fa9b12fd7d1c33ce62b875 into 861859c95bf74d28ab8f1b7b1f994ccf(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:49,265 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/B of abf9890bb5fa9b12fd7d1c33ce62b875 into 5ceebd8f87c6425385bb2bb23e5d54ee(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:49,265 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:49,265 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:49,265 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/B, priority=13, startTime=1731394128836; duration=0sec 2024-11-12T06:48:49,265 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/A, priority=13, startTime=1731394128836; duration=0sec 2024-11-12T06:48:49,265 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:49,265 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:B 2024-11-12T06:48:49,265 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:48:49,266 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:A 2024-11-12T06:48:49,266 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:48:49,267 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:48:49,267 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): abf9890bb5fa9b12fd7d1c33ce62b875/C is initiating minor compaction (all files) 2024-11-12T06:48:49,267 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abf9890bb5fa9b12fd7d1c33ce62b875/C in TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:49,267 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ae0a3a82fa4c447fbd84d48e8386d83d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/5a7ec4624817417398c30dcda6d997f4, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/c1deceb3da5544cc86a147d946964fb1] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp, totalSize=34.5 K 2024-11-12T06:48:49,267 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting ae0a3a82fa4c447fbd84d48e8386d83d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731394125028 2024-11-12T06:48:49,267 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a7ec4624817417398c30dcda6d997f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1731394126700 2024-11-12T06:48:49,268 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting c1deceb3da5544cc86a147d946964fb1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=589, earliestPutTs=1731394127988 2024-11-12T06:48:49,273 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abf9890bb5fa9b12fd7d1c33ce62b875#C#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:48:49,274 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/b3812ef6f67149d6a3320b306544cc94 is 50, key is test_row_0/C:col10/1731394126700/Put/seqid=0 2024-11-12T06:48:49,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742144_1320 (size=13255) 2024-11-12T06:48:49,341 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/4451198e05ac4abbbcb3002e2b55bd67 2024-11-12T06:48:49,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/449da6d64607437883995d03e0bb67c0 is 50, key is test_row_0/B:col10/1731394128048/Put/seqid=0 2024-11-12T06:48:49,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742145_1321 (size=12301) 2024-11-12T06:48:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T06:48:49,682 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/b3812ef6f67149d6a3320b306544cc94 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/b3812ef6f67149d6a3320b306544cc94 2024-11-12T06:48:49,687 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abf9890bb5fa9b12fd7d1c33ce62b875/C of abf9890bb5fa9b12fd7d1c33ce62b875 into b3812ef6f67149d6a3320b306544cc94(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:48:49,687 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:49,687 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., storeName=abf9890bb5fa9b12fd7d1c33ce62b875/C, priority=13, startTime=1731394128836; duration=0sec 2024-11-12T06:48:49,687 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:48:49,687 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abf9890bb5fa9b12fd7d1c33ce62b875:C 2024-11-12T06:48:49,761 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/449da6d64607437883995d03e0bb67c0 2024-11-12T06:48:49,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/3c41fec793374ee3a764c5d185beece4 is 50, key is test_row_0/C:col10/1731394128048/Put/seqid=0 2024-11-12T06:48:49,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742146_1322 (size=12301) 2024-11-12T06:48:50,179 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/3c41fec793374ee3a764c5d185beece4 2024-11-12T06:48:50,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/4451198e05ac4abbbcb3002e2b55bd67 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/4451198e05ac4abbbcb3002e2b55bd67 2024-11-12T06:48:50,195 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/4451198e05ac4abbbcb3002e2b55bd67, entries=150, sequenceid=626, filesize=12.0 K 2024-11-12T06:48:50,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/449da6d64607437883995d03e0bb67c0 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/449da6d64607437883995d03e0bb67c0 2024-11-12T06:48:50,200 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/449da6d64607437883995d03e0bb67c0, entries=150, sequenceid=626, filesize=12.0 K 2024-11-12T06:48:50,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/3c41fec793374ee3a764c5d185beece4 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/3c41fec793374ee3a764c5d185beece4 2024-11-12T06:48:50,203 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/3c41fec793374ee3a764c5d185beece4, entries=150, sequenceid=626, filesize=12.0 K 2024-11-12T06:48:50,204 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=20.13 KB/20610 for abf9890bb5fa9b12fd7d1c33ce62b875 in 1274ms, sequenceid=626, compaction requested=false 2024-11-12T06:48:50,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for abf9890bb5fa9b12fd7d1c33ce62b875: 2024-11-12T06:48:50,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:50,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-12T06:48:50,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-12T06:48:50,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-12T06:48:50,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8850 sec 2024-11-12T06:48:50,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.8890 sec 2024-11-12T06:48:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T06:48:50,467 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:51,502 INFO [Thread-1086 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1086. 2024-11-12T06:48:51,503 DEBUG [Thread-1086 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:48:51,503 DEBUG [Thread-1086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:51,503 DEBUG [Thread-1086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:51,503 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:48:51,519 INFO [Thread-1088 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1088. 2024-11-12T06:48:51,519 DEBUG [Thread-1088 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:48:51,519 DEBUG [Thread-1088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:51,519 DEBUG [Thread-1088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:51,519 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:48:51,519 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T06:48:51,519 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 152 2024-11-12T06:48:51,519 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-12T06:48:51,519 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-12T06:48:51,520 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 122 2024-11-12T06:48:51,520 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 139 2024-11-12T06:48:51,520 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T06:48:51,520 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3473 2024-11-12T06:48:51,520 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3283 2024-11-12T06:48:51,520 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3268 2024-11-12T06:48:51,520 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3478 2024-11-12T06:48:51,520 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3280 2024-11-12T06:48:51,520 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T06:48:51,520 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T06:48:51,520 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T06:48:51,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:51,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:51,520 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:48:51,522 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T06:48:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T06:48:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-12T06:48:51,528 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394131527"}]},"ts":"1731394131527"} 2024-11-12T06:48:51,530 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T06:48:51,530 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T06:48:51,532 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T06:48:51,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=abf9890bb5fa9b12fd7d1c33ce62b875, UNASSIGN}] 2024-11-12T06:48:51,536 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=abf9890bb5fa9b12fd7d1c33ce62b875, UNASSIGN 2024-11-12T06:48:51,536 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=abf9890bb5fa9b12fd7d1c33ce62b875, regionState=CLOSING, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:51,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=abf9890bb5fa9b12fd7d1c33ce62b875, UNASSIGN because future has completed 2024-11-12T06:48:51,538 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T06:48:51,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622}] 2024-11-12T06:48:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-12T06:48:51,692 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:51,692 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T06:48:51,692 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing abf9890bb5fa9b12fd7d1c33ce62b875, disabling compactions & flushes 2024-11-12T06:48:51,692 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:51,692 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:51,693 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. after waiting 0 ms 2024-11-12T06:48:51,693 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:51,693 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing abf9890bb5fa9b12fd7d1c33ce62b875 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-12T06:48:51,693 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=A 2024-11-12T06:48:51,694 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:51,694 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=B 2024-11-12T06:48:51,694 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:51,694 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK abf9890bb5fa9b12fd7d1c33ce62b875, store=C 2024-11-12T06:48:51,694 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:51,703 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/4122364b086c4e159f2eee73b97552de is 50, key is test_row_0/A:col10/1731394131499/Put/seqid=0 2024-11-12T06:48:51,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742147_1323 (size=12301) 2024-11-12T06:48:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-12T06:48:52,108 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=637 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/4122364b086c4e159f2eee73b97552de 2024-11-12T06:48:52,121 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/d01230eb17a24f3a87733bf54325bc1b is 50, key is test_row_0/B:col10/1731394131499/Put/seqid=0 2024-11-12T06:48:52,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742148_1324 (size=12301) 2024-11-12T06:48:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-12T06:48:52,526 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=637 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/d01230eb17a24f3a87733bf54325bc1b 2024-11-12T06:48:52,537 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/4d4ec765de544bf4a26f7b55b3a55432 is 50, key is test_row_0/C:col10/1731394131499/Put/seqid=0 2024-11-12T06:48:52,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742149_1325 (size=12301) 2024-11-12T06:48:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-12T06:48:52,942 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=637 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/4d4ec765de544bf4a26f7b55b3a55432 2024-11-12T06:48:52,955 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/A/4122364b086c4e159f2eee73b97552de as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/4122364b086c4e159f2eee73b97552de 2024-11-12T06:48:52,959 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/4122364b086c4e159f2eee73b97552de, entries=150, sequenceid=637, filesize=12.0 K 2024-11-12T06:48:52,960 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/B/d01230eb17a24f3a87733bf54325bc1b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/d01230eb17a24f3a87733bf54325bc1b 2024-11-12T06:48:52,963 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/d01230eb17a24f3a87733bf54325bc1b, entries=150, sequenceid=637, filesize=12.0 K 2024-11-12T06:48:52,964 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/.tmp/C/4d4ec765de544bf4a26f7b55b3a55432 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/4d4ec765de544bf4a26f7b55b3a55432 2024-11-12T06:48:52,967 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/4d4ec765de544bf4a26f7b55b3a55432, entries=150, sequenceid=637, filesize=12.0 K 2024-11-12T06:48:52,968 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for abf9890bb5fa9b12fd7d1c33ce62b875 in 1275ms, sequenceid=637, compaction requested=true 2024-11-12T06:48:52,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/dbc6fdb2cfc641039aed00a8c8314886, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/0c4c9d2c4b5b4d18aae3493529435f9b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/d5312f27e25a4ade9e7af8d2c1535907, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/c4c26e843e30476da167a6ceff0e00d7, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/1f3806be1e1e4c9882d7b90347026d9b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/266cc74af26b4d2cba078a76b1416ae0, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/67cc1cde43aa41888653d2738c4b1a53, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/d62df8c78585496f8b703744fd7f4fa6, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/bc8a3d093da14f8e85324c0b56ec4f9b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/a1ff2ce4c8994712937e6fb343b5e55c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/944a414fc7674f3ea8a8ea57538c09d4, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/3326e57655f445ebba43c14b1f7cb95e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/b52a77c015364bc5b1223dc3e3cdb994, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/af5e8a65fd434f0ca1beeffc71a0f08f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/7abacae8d3ad4b7ea32f896d0bd7130d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/63dea34c8a1547219ee2fd5610aec9b2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/4b8c252200694228acd3d4e50a227b48, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/0112d2b8b97c48b7b4e1628a0dbf3ebb, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/d1372216db4140bb8341032fc235053a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/11f2c3c79367409f96047a1b60bf8075, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/bb52239391d14509be4777c83e1bd7b2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/f6f4fbd12c1d42eba4cf3b621612d68a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/2a91978d8a124110a7b539c3b615bcc2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/4c8d5e0fb6814fb38ebe35a917cda3a9, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/5b287c969d66439ab438c50664f64a71, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/7bb4b1e7a68f4aa4878fb63b1ee617d2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/76e5a833de00408880b3aad5414f810c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/f3e5f5dfe0c348c4a48333dddde7a9a6] to archive 2024-11-12T06:48:52,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T06:48:52,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/dbc6fdb2cfc641039aed00a8c8314886 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/dbc6fdb2cfc641039aed00a8c8314886 2024-11-12T06:48:52,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/0c4c9d2c4b5b4d18aae3493529435f9b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/0c4c9d2c4b5b4d18aae3493529435f9b 2024-11-12T06:48:52,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/d5312f27e25a4ade9e7af8d2c1535907 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/d5312f27e25a4ade9e7af8d2c1535907 2024-11-12T06:48:52,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/c4c26e843e30476da167a6ceff0e00d7 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/c4c26e843e30476da167a6ceff0e00d7 2024-11-12T06:48:52,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/1f3806be1e1e4c9882d7b90347026d9b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/1f3806be1e1e4c9882d7b90347026d9b 2024-11-12T06:48:52,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/266cc74af26b4d2cba078a76b1416ae0 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/266cc74af26b4d2cba078a76b1416ae0 2024-11-12T06:48:52,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/67cc1cde43aa41888653d2738c4b1a53 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/67cc1cde43aa41888653d2738c4b1a53 2024-11-12T06:48:52,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/d62df8c78585496f8b703744fd7f4fa6 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/d62df8c78585496f8b703744fd7f4fa6 2024-11-12T06:48:52,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/bc8a3d093da14f8e85324c0b56ec4f9b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/bc8a3d093da14f8e85324c0b56ec4f9b 2024-11-12T06:48:52,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/a1ff2ce4c8994712937e6fb343b5e55c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/a1ff2ce4c8994712937e6fb343b5e55c 2024-11-12T06:48:52,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/944a414fc7674f3ea8a8ea57538c09d4 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/944a414fc7674f3ea8a8ea57538c09d4 2024-11-12T06:48:52,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/3326e57655f445ebba43c14b1f7cb95e to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/3326e57655f445ebba43c14b1f7cb95e 2024-11-12T06:48:52,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/b52a77c015364bc5b1223dc3e3cdb994 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/b52a77c015364bc5b1223dc3e3cdb994 2024-11-12T06:48:52,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/af5e8a65fd434f0ca1beeffc71a0f08f to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/af5e8a65fd434f0ca1beeffc71a0f08f 2024-11-12T06:48:52,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/7abacae8d3ad4b7ea32f896d0bd7130d to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/7abacae8d3ad4b7ea32f896d0bd7130d 2024-11-12T06:48:52,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/63dea34c8a1547219ee2fd5610aec9b2 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/63dea34c8a1547219ee2fd5610aec9b2 2024-11-12T06:48:52,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/4b8c252200694228acd3d4e50a227b48 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/4b8c252200694228acd3d4e50a227b48 2024-11-12T06:48:52,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/0112d2b8b97c48b7b4e1628a0dbf3ebb to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/0112d2b8b97c48b7b4e1628a0dbf3ebb 2024-11-12T06:48:52,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/d1372216db4140bb8341032fc235053a to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/d1372216db4140bb8341032fc235053a 2024-11-12T06:48:52,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/11f2c3c79367409f96047a1b60bf8075 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/11f2c3c79367409f96047a1b60bf8075 2024-11-12T06:48:52,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/bb52239391d14509be4777c83e1bd7b2 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/bb52239391d14509be4777c83e1bd7b2 2024-11-12T06:48:52,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/f6f4fbd12c1d42eba4cf3b621612d68a to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/f6f4fbd12c1d42eba4cf3b621612d68a 2024-11-12T06:48:52,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/2a91978d8a124110a7b539c3b615bcc2 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/2a91978d8a124110a7b539c3b615bcc2 2024-11-12T06:48:52,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/4c8d5e0fb6814fb38ebe35a917cda3a9 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/4c8d5e0fb6814fb38ebe35a917cda3a9 2024-11-12T06:48:52,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/5b287c969d66439ab438c50664f64a71 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/5b287c969d66439ab438c50664f64a71 2024-11-12T06:48:52,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/7bb4b1e7a68f4aa4878fb63b1ee617d2 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/7bb4b1e7a68f4aa4878fb63b1ee617d2 2024-11-12T06:48:52,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/76e5a833de00408880b3aad5414f810c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/76e5a833de00408880b3aad5414f810c 2024-11-12T06:48:52,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/f3e5f5dfe0c348c4a48333dddde7a9a6 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/f3e5f5dfe0c348c4a48333dddde7a9a6 2024-11-12T06:48:52,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/b958513f0367416b99065a8181753372, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/9f3d9000e72c411b8d54f8e662c37b4c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/e7edc347256f4f1ba3d5ef0818799f6b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/9032ff40632541bdbb8eeaf213bab2ce, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/d7a4b81cf24d4839a908a98a9c7b008c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/21973ceb9e0f49718209fd2301aafc79, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/ea9b076238bf41a2a0c17dedb9e308f9, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/017c3570ea654bca8f5690cca712a40e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/6d966b9d979846b9bee9a071ea2c819f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/9df3092c249e42bb852d3b8134c2a04b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/4b897cb89cc44a4ab7b1d137e2947949, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/510aefbc86c94c5a937e4336751c827a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/fd41aaffef4e452eb03805d419dc1326, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/adc98b76ed044eb289aaefae48a62f82, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/18903bede2894ca28a74ab8587a0aa51, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/fc138462d0ff45adb978423a555e47ed, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/07f1364b394444f693098cbea19a207c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/d7276369fe854d298d77d7046aad9282, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/174c4fb27a334bdfb6723a332cfb4908, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/a1bbc5317df34bc7b189888c19e9bc94, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/3e288ae833664252974eab07b56cf2a2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/009cef0895084fd9aca58f603a3bb66b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/58cf427ac96f49f292d86c71f7dc7656, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/ad55e00231874dddab5c3d66cac0515a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/315a0bcb16494036af8d7441e9cfaeb9, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/0a465946a1f64cf59a32cfc43b3aeb14, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/86c199f7dca341d9a8a0a23c93e46ccb, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/146607a57e8d4abdb3f4afa359370373] to archive 2024-11-12T06:48:52,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T06:48:52,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/b958513f0367416b99065a8181753372 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/b958513f0367416b99065a8181753372 2024-11-12T06:48:52,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/9f3d9000e72c411b8d54f8e662c37b4c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/9f3d9000e72c411b8d54f8e662c37b4c 2024-11-12T06:48:52,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/e7edc347256f4f1ba3d5ef0818799f6b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/e7edc347256f4f1ba3d5ef0818799f6b 2024-11-12T06:48:53,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/9032ff40632541bdbb8eeaf213bab2ce to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/9032ff40632541bdbb8eeaf213bab2ce 2024-11-12T06:48:53,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/d7a4b81cf24d4839a908a98a9c7b008c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/d7a4b81cf24d4839a908a98a9c7b008c 2024-11-12T06:48:53,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/21973ceb9e0f49718209fd2301aafc79 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/21973ceb9e0f49718209fd2301aafc79 2024-11-12T06:48:53,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/ea9b076238bf41a2a0c17dedb9e308f9 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/ea9b076238bf41a2a0c17dedb9e308f9 2024-11-12T06:48:53,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/017c3570ea654bca8f5690cca712a40e to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/017c3570ea654bca8f5690cca712a40e 2024-11-12T06:48:53,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/6d966b9d979846b9bee9a071ea2c819f to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/6d966b9d979846b9bee9a071ea2c819f 2024-11-12T06:48:53,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/9df3092c249e42bb852d3b8134c2a04b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/9df3092c249e42bb852d3b8134c2a04b 2024-11-12T06:48:53,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/4b897cb89cc44a4ab7b1d137e2947949 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/4b897cb89cc44a4ab7b1d137e2947949 2024-11-12T06:48:53,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/510aefbc86c94c5a937e4336751c827a to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/510aefbc86c94c5a937e4336751c827a 2024-11-12T06:48:53,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/fd41aaffef4e452eb03805d419dc1326 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/fd41aaffef4e452eb03805d419dc1326 2024-11-12T06:48:53,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/adc98b76ed044eb289aaefae48a62f82 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/adc98b76ed044eb289aaefae48a62f82 2024-11-12T06:48:53,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/18903bede2894ca28a74ab8587a0aa51 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/18903bede2894ca28a74ab8587a0aa51 2024-11-12T06:48:53,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/fc138462d0ff45adb978423a555e47ed to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/fc138462d0ff45adb978423a555e47ed 2024-11-12T06:48:53,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/07f1364b394444f693098cbea19a207c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/07f1364b394444f693098cbea19a207c 2024-11-12T06:48:53,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/d7276369fe854d298d77d7046aad9282 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/d7276369fe854d298d77d7046aad9282 2024-11-12T06:48:53,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/174c4fb27a334bdfb6723a332cfb4908 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/174c4fb27a334bdfb6723a332cfb4908 2024-11-12T06:48:53,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/a1bbc5317df34bc7b189888c19e9bc94 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/a1bbc5317df34bc7b189888c19e9bc94 2024-11-12T06:48:53,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/3e288ae833664252974eab07b56cf2a2 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/3e288ae833664252974eab07b56cf2a2 2024-11-12T06:48:53,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/009cef0895084fd9aca58f603a3bb66b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/009cef0895084fd9aca58f603a3bb66b 2024-11-12T06:48:53,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/58cf427ac96f49f292d86c71f7dc7656 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/58cf427ac96f49f292d86c71f7dc7656 2024-11-12T06:48:53,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/ad55e00231874dddab5c3d66cac0515a to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/ad55e00231874dddab5c3d66cac0515a 2024-11-12T06:48:53,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/315a0bcb16494036af8d7441e9cfaeb9 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/315a0bcb16494036af8d7441e9cfaeb9 2024-11-12T06:48:53,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/0a465946a1f64cf59a32cfc43b3aeb14 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/0a465946a1f64cf59a32cfc43b3aeb14 2024-11-12T06:48:53,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/86c199f7dca341d9a8a0a23c93e46ccb to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/86c199f7dca341d9a8a0a23c93e46ccb 2024-11-12T06:48:53,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/146607a57e8d4abdb3f4afa359370373 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/146607a57e8d4abdb3f4afa359370373 2024-11-12T06:48:53,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/48295ba63bce44dca13b97c9a748667d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/29d8003dce8e4c4fbe16625d0e15c02b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/a56c503443de46e7acbb03a383a8735c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/5c735e4bf6ac42fe9edffac342002b50, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ae019e9a739d4f76bb2a0d4a24c296eb, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/fe83f5f8acca4d8ba30b0731b4beecf9, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/95b2cb9068bb428c834d48dbd4667c4b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/3eaf431f32b44fab8de34e708e830ad7, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ca273260399d4a9d922ed852527ebcae, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/e9b1957e32b1498f9a6963587c371d44, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ad91a8a1ca62452c87d8604123595a6c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/f28eb0fecaa24a68890c2d362af88b26, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/059c5274fb0e4388b09ec654cf6335e5, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/21dcf50fc4fe4108a4db28361400edc1, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/6e27a907b6974cb2a5898e7cc8db944e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ebb98ffb615c4361b1f7023f3d20b6a8, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/9036e1ca6c6e40e49e6c831d19707215, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/af93507c7e104e12b442fc15456ccc5d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/86e2616b69244993b5aceeb8cdbb17ff, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/0697008601694113975a4c03d642a524, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/166093e5b6ad46d2a83ff8bd89d31b63, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/35565ebe50c248909f9d4e891d8bdaf0, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/3924f42139984dcca463ac21f68075ad, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/1da3750e19a447f885bd7d8d1c2ef75f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ae0a3a82fa4c447fbd84d48e8386d83d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/2bdc98965d5741449ffb52769204d0a6, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/5a7ec4624817417398c30dcda6d997f4, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/c1deceb3da5544cc86a147d946964fb1] to archive 2024-11-12T06:48:53,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T06:48:53,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/48295ba63bce44dca13b97c9a748667d to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/48295ba63bce44dca13b97c9a748667d 2024-11-12T06:48:53,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/29d8003dce8e4c4fbe16625d0e15c02b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/29d8003dce8e4c4fbe16625d0e15c02b 2024-11-12T06:48:53,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/a56c503443de46e7acbb03a383a8735c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/a56c503443de46e7acbb03a383a8735c 2024-11-12T06:48:53,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/5c735e4bf6ac42fe9edffac342002b50 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/5c735e4bf6ac42fe9edffac342002b50 2024-11-12T06:48:53,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ae019e9a739d4f76bb2a0d4a24c296eb to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ae019e9a739d4f76bb2a0d4a24c296eb 2024-11-12T06:48:53,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/fe83f5f8acca4d8ba30b0731b4beecf9 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/fe83f5f8acca4d8ba30b0731b4beecf9 2024-11-12T06:48:53,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/95b2cb9068bb428c834d48dbd4667c4b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/95b2cb9068bb428c834d48dbd4667c4b 2024-11-12T06:48:53,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/3eaf431f32b44fab8de34e708e830ad7 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/3eaf431f32b44fab8de34e708e830ad7 2024-11-12T06:48:53,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ca273260399d4a9d922ed852527ebcae to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ca273260399d4a9d922ed852527ebcae 2024-11-12T06:48:53,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/e9b1957e32b1498f9a6963587c371d44 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/e9b1957e32b1498f9a6963587c371d44 2024-11-12T06:48:53,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ad91a8a1ca62452c87d8604123595a6c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ad91a8a1ca62452c87d8604123595a6c 2024-11-12T06:48:53,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/f28eb0fecaa24a68890c2d362af88b26 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/f28eb0fecaa24a68890c2d362af88b26 2024-11-12T06:48:53,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/059c5274fb0e4388b09ec654cf6335e5 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/059c5274fb0e4388b09ec654cf6335e5 2024-11-12T06:48:53,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/21dcf50fc4fe4108a4db28361400edc1 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/21dcf50fc4fe4108a4db28361400edc1 2024-11-12T06:48:53,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/6e27a907b6974cb2a5898e7cc8db944e to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/6e27a907b6974cb2a5898e7cc8db944e 2024-11-12T06:48:53,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ebb98ffb615c4361b1f7023f3d20b6a8 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ebb98ffb615c4361b1f7023f3d20b6a8 2024-11-12T06:48:53,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/9036e1ca6c6e40e49e6c831d19707215 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/9036e1ca6c6e40e49e6c831d19707215 2024-11-12T06:48:53,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/af93507c7e104e12b442fc15456ccc5d to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/af93507c7e104e12b442fc15456ccc5d 2024-11-12T06:48:53,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/86e2616b69244993b5aceeb8cdbb17ff to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/86e2616b69244993b5aceeb8cdbb17ff 2024-11-12T06:48:53,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/0697008601694113975a4c03d642a524 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/0697008601694113975a4c03d642a524 2024-11-12T06:48:53,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/166093e5b6ad46d2a83ff8bd89d31b63 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/166093e5b6ad46d2a83ff8bd89d31b63 2024-11-12T06:48:53,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/35565ebe50c248909f9d4e891d8bdaf0 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/35565ebe50c248909f9d4e891d8bdaf0 2024-11-12T06:48:53,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/3924f42139984dcca463ac21f68075ad to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/3924f42139984dcca463ac21f68075ad 2024-11-12T06:48:53,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/1da3750e19a447f885bd7d8d1c2ef75f to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/1da3750e19a447f885bd7d8d1c2ef75f 2024-11-12T06:48:53,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ae0a3a82fa4c447fbd84d48e8386d83d to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/ae0a3a82fa4c447fbd84d48e8386d83d 2024-11-12T06:48:53,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/2bdc98965d5741449ffb52769204d0a6 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/2bdc98965d5741449ffb52769204d0a6 2024-11-12T06:48:53,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/5a7ec4624817417398c30dcda6d997f4 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/5a7ec4624817417398c30dcda6d997f4 2024-11-12T06:48:53,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/c1deceb3da5544cc86a147d946964fb1 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/c1deceb3da5544cc86a147d946964fb1 2024-11-12T06:48:53,047 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/recovered.edits/640.seqid, newMaxSeqId=640, maxSeqId=1 2024-11-12T06:48:53,048 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. 2024-11-12T06:48:53,048 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for abf9890bb5fa9b12fd7d1c33ce62b875: Waiting for close lock at 1731394131692Running coprocessor pre-close hooks at 1731394131692Disabling compacts and flushes for region at 1731394131692Disabling writes for close at 1731394131693 (+1 ms)Obtaining lock to block concurrent updates at 1731394131693Preparing flush snapshotting stores in abf9890bb5fa9b12fd7d1c33ce62b875 at 1731394131693Finished memstore snapshotting TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731394131694 (+1 ms)Flushing stores of TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875. at 1731394131695 (+1 ms)Flushing abf9890bb5fa9b12fd7d1c33ce62b875/A: creating writer at 1731394131695Flushing abf9890bb5fa9b12fd7d1c33ce62b875/A: appending metadata at 1731394131702 (+7 ms)Flushing abf9890bb5fa9b12fd7d1c33ce62b875/A: closing flushed file at 1731394131702Flushing abf9890bb5fa9b12fd7d1c33ce62b875/B: creating writer at 1731394132117 (+415 ms)Flushing abf9890bb5fa9b12fd7d1c33ce62b875/B: appending metadata at 1731394132120 (+3 ms)Flushing abf9890bb5fa9b12fd7d1c33ce62b875/B: closing flushed file at 1731394132120Flushing abf9890bb5fa9b12fd7d1c33ce62b875/C: creating writer at 1731394132532 (+412 ms)Flushing abf9890bb5fa9b12fd7d1c33ce62b875/C: appending metadata at 1731394132536 (+4 ms)Flushing abf9890bb5fa9b12fd7d1c33ce62b875/C: closing flushed file at 1731394132536Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58ced533: reopening flushed file at 1731394132954 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65d943fa: reopening flushed file at 1731394132959 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d1d0652: reopening flushed file at 1731394132963 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for abf9890bb5fa9b12fd7d1c33ce62b875 in 1275ms, sequenceid=637, compaction requested=true at 1731394132968 (+5 ms)Writing region close event to WAL at 1731394133045 (+77 ms)Running coprocessor post-close hooks at 1731394133048 (+3 ms)Closed at 1731394133048 2024-11-12T06:48:53,050 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:53,050 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=abf9890bb5fa9b12fd7d1c33ce62b875, regionState=CLOSED 2024-11-12T06:48:53,051 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 because future has completed 2024-11-12T06:48:53,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-12T06:48:53,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure abf9890bb5fa9b12fd7d1c33ce62b875, server=9f0beee3d6b6,38777,1731394047622 in 1.5140 sec 2024-11-12T06:48:53,055 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-12T06:48:53,055 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=abf9890bb5fa9b12fd7d1c33ce62b875, UNASSIGN in 1.5200 sec 2024-11-12T06:48:53,057 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-12T06:48:53,057 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5240 sec 2024-11-12T06:48:53,058 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394133058"}]},"ts":"1731394133058"} 2024-11-12T06:48:53,059 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T06:48:53,059 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T06:48:53,060 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5380 sec 2024-11-12T06:48:53,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-12T06:48:53,679 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:53,680 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T06:48:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:48:53,685 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:48:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-12T06:48:53,688 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:48:53,693 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:53,696 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A, FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B, FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C, FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/recovered.edits] 2024-11-12T06:48:53,699 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/4122364b086c4e159f2eee73b97552de to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/4122364b086c4e159f2eee73b97552de 2024-11-12T06:48:53,700 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/4451198e05ac4abbbcb3002e2b55bd67 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/4451198e05ac4abbbcb3002e2b55bd67 2024-11-12T06:48:53,702 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/861859c95bf74d28ab8f1b7b1f994ccf to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/A/861859c95bf74d28ab8f1b7b1f994ccf 2024-11-12T06:48:53,705 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/449da6d64607437883995d03e0bb67c0 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/449da6d64607437883995d03e0bb67c0 2024-11-12T06:48:53,707 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/5ceebd8f87c6425385bb2bb23e5d54ee to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/5ceebd8f87c6425385bb2bb23e5d54ee 2024-11-12T06:48:53,708 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/d01230eb17a24f3a87733bf54325bc1b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/B/d01230eb17a24f3a87733bf54325bc1b 2024-11-12T06:48:53,711 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/3c41fec793374ee3a764c5d185beece4 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/3c41fec793374ee3a764c5d185beece4 2024-11-12T06:48:53,713 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/4d4ec765de544bf4a26f7b55b3a55432 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/4d4ec765de544bf4a26f7b55b3a55432 2024-11-12T06:48:53,714 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/b3812ef6f67149d6a3320b306544cc94 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/C/b3812ef6f67149d6a3320b306544cc94 2024-11-12T06:48:53,718 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/recovered.edits/640.seqid to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875/recovered.edits/640.seqid 2024-11-12T06:48:53,718 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/abf9890bb5fa9b12fd7d1c33ce62b875 2024-11-12T06:48:53,719 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T06:48:53,721 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:48:53,725 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T06:48:53,727 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T06:48:53,728 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:48:53,728 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T06:48:53,728 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731394133728"}]},"ts":"9223372036854775807"} 2024-11-12T06:48:53,730 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T06:48:53,730 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => abf9890bb5fa9b12fd7d1c33ce62b875, NAME => 'TestAcidGuarantees,,1731394106744.abf9890bb5fa9b12fd7d1c33ce62b875.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T06:48:53,730 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T06:48:53,730 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731394133730"}]},"ts":"9223372036854775807"} 2024-11-12T06:48:53,732 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T06:48:53,733 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:48:53,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-11-12T06:48:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-12T06:48:53,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T06:48:53,799 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:53,812 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251 (was 252), OpenFileDescriptor=457 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=348 (was 313) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6655 (was 6752) 2024-11-12T06:48:53,822 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=348, ProcessCount=11, AvailableMemoryMB=6654 2024-11-12T06:48:53,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T06:48:53,824 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T06:48:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T06:48:53,827 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T06:48:53,827 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:53,827 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-11-12T06:48:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T06:48:53,828 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T06:48:53,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742150_1326 (size=1061) 2024-11-12T06:48:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T06:48:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T06:48:54,241 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e 2024-11-12T06:48:54,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742151_1327 (size=53) 2024-11-12T06:48:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T06:48:54,650 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T06:48:54,651 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a474a06785ead13d749016bf6309b74c, disabling compactions & flushes 2024-11-12T06:48:54,651 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:54,651 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:54,651 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. after waiting 0 ms 2024-11-12T06:48:54,651 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:54,651 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:54,651 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a474a06785ead13d749016bf6309b74c: Waiting for close lock at 1731394134650Disabling compacts and flushes for region at 1731394134650Disabling writes for close at 1731394134651 (+1 ms)Writing region close event to WAL at 1731394134651Closed at 1731394134651 2024-11-12T06:48:54,654 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T06:48:54,655 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731394134654"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731394134654"}]},"ts":"1731394134654"} 2024-11-12T06:48:54,657 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T06:48:54,658 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T06:48:54,658 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394134658"}]},"ts":"1731394134658"} 2024-11-12T06:48:54,660 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T06:48:54,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a474a06785ead13d749016bf6309b74c, ASSIGN}] 2024-11-12T06:48:54,661 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a474a06785ead13d749016bf6309b74c, ASSIGN 2024-11-12T06:48:54,662 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a474a06785ead13d749016bf6309b74c, ASSIGN; state=OFFLINE, location=9f0beee3d6b6,38777,1731394047622; forceNewPlan=false, retain=false 2024-11-12T06:48:54,813 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=a474a06785ead13d749016bf6309b74c, regionState=OPENING, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:54,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a474a06785ead13d749016bf6309b74c, ASSIGN because future has completed 2024-11-12T06:48:54,817 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622}] 2024-11-12T06:48:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T06:48:54,981 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:54,981 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} 2024-11-12T06:48:54,982 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:54,982 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T06:48:54,982 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:54,982 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:54,983 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:54,984 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:48:54,985 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a474a06785ead13d749016bf6309b74c columnFamilyName A 2024-11-12T06:48:54,985 DEBUG [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:54,985 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] regionserver.HStore(327): Store=a474a06785ead13d749016bf6309b74c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:48:54,985 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:54,986 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:48:54,986 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a474a06785ead13d749016bf6309b74c columnFamilyName B 2024-11-12T06:48:54,986 DEBUG [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:54,987 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] regionserver.HStore(327): Store=a474a06785ead13d749016bf6309b74c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:48:54,987 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:54,988 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:48:54,988 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a474a06785ead13d749016bf6309b74c columnFamilyName C 2024-11-12T06:48:54,988 DEBUG [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:54,988 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] regionserver.HStore(327): Store=a474a06785ead13d749016bf6309b74c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:48:54,989 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:54,989 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:54,989 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:54,990 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:54,990 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:54,990 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:54,991 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T06:48:54,992 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:54,994 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T06:48:54,994 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened a474a06785ead13d749016bf6309b74c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63877334, jitterRate=-0.04815354943275452}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T06:48:54,994 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:54,995 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for a474a06785ead13d749016bf6309b74c: Running coprocessor pre-open hook at 1731394134982Writing region info on filesystem at 1731394134982Initializing all the Stores at 1731394134983 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394134983Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394134983Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394134983Cleaning up temporary data from old regions at 1731394134991 (+8 ms)Running coprocessor post-open hooks at 1731394134994 (+3 ms)Region opened successfully at 1731394134995 (+1 ms) 2024-11-12T06:48:54,996 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., pid=91, masterSystemTime=1731394134972 2024-11-12T06:48:54,997 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:54,997 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:54,998 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=a474a06785ead13d749016bf6309b74c, regionState=OPEN, openSeqNum=2, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:54,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 because future has completed 2024-11-12T06:48:55,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-12T06:48:55,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 in 183 msec 2024-11-12T06:48:55,004 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-12T06:48:55,004 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a474a06785ead13d749016bf6309b74c, ASSIGN in 342 msec 2024-11-12T06:48:55,005 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T06:48:55,005 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394135005"}]},"ts":"1731394135005"} 2024-11-12T06:48:55,007 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T06:48:55,008 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T06:48:55,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1840 sec 2024-11-12T06:48:55,969 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T06:48:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T06:48:55,978 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:55,981 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db9998b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:55,981 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:55,982 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:55,984 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:55,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:55,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73439d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:55,986 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:55,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64451ca9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:55,987 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:55,989 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:55,990 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T06:48:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 9f0beee3d6b6,43031,1731394046864 2024-11-12T06:48:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@682e547e 2024-11-12T06:48:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T06:48:55,992 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T06:48:55,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T06:48:55,994 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T06:48:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-12T06:48:56,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742152_1328 (size=1097) 2024-11-12T06:48:56,414 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-12T06:48:56,414 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-12T06:48:56,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T06:48:56,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a474a06785ead13d749016bf6309b74c, REOPEN/MOVE}] 2024-11-12T06:48:56,420 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a474a06785ead13d749016bf6309b74c, REOPEN/MOVE 2024-11-12T06:48:56,420 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=a474a06785ead13d749016bf6309b74c, regionState=CLOSING, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:56,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a474a06785ead13d749016bf6309b74c, REOPEN/MOVE because future has completed 2024-11-12T06:48:56,422 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T06:48:56,422 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622}] 2024-11-12T06:48:56,575 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:56,575 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T06:48:56,575 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing a474a06785ead13d749016bf6309b74c, disabling compactions & flushes 2024-11-12T06:48:56,576 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:56,576 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:56,576 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. after waiting 0 ms 2024-11-12T06:48:56,576 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:56,583 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T06:48:56,584 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:56,584 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for a474a06785ead13d749016bf6309b74c: Waiting for close lock at 1731394136575Running coprocessor pre-close hooks at 1731394136575Disabling compacts and flushes for region at 1731394136575Disabling writes for close at 1731394136576 (+1 ms)Writing region close event to WAL at 1731394136577 (+1 ms)Running coprocessor post-close hooks at 1731394136584 (+7 ms)Closed at 1731394136584 2024-11-12T06:48:56,584 WARN [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: a474a06785ead13d749016bf6309b74c to self. 2024-11-12T06:48:56,586 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:56,587 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=a474a06785ead13d749016bf6309b74c, regionState=CLOSED 2024-11-12T06:48:56,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 because future has completed 2024-11-12T06:48:56,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-12T06:48:56,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 in 167 msec 2024-11-12T06:48:56,592 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a474a06785ead13d749016bf6309b74c, REOPEN/MOVE; state=CLOSED, location=9f0beee3d6b6,38777,1731394047622; forceNewPlan=false, retain=true 2024-11-12T06:48:56,743 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=a474a06785ead13d749016bf6309b74c, regionState=OPENING, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:56,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a474a06785ead13d749016bf6309b74c, REOPEN/MOVE because future has completed 2024-11-12T06:48:56,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622}] 2024-11-12T06:48:56,909 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:56,909 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} 2024-11-12T06:48:56,910 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:56,910 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T06:48:56,910 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:56,910 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:56,914 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:56,915 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:48:56,915 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a474a06785ead13d749016bf6309b74c columnFamilyName A 2024-11-12T06:48:56,916 DEBUG [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:56,916 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] regionserver.HStore(327): Store=a474a06785ead13d749016bf6309b74c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:48:56,917 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:56,917 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:48:56,917 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a474a06785ead13d749016bf6309b74c columnFamilyName B 2024-11-12T06:48:56,917 DEBUG [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:56,918 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] regionserver.HStore(327): Store=a474a06785ead13d749016bf6309b74c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:48:56,918 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:56,918 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:48:56,918 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a474a06785ead13d749016bf6309b74c columnFamilyName C 2024-11-12T06:48:56,918 DEBUG [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:56,919 INFO [StoreOpener-a474a06785ead13d749016bf6309b74c-1 {}] regionserver.HStore(327): Store=a474a06785ead13d749016bf6309b74c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:48:56,919 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:56,919 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:56,919 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:56,920 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:56,921 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:56,921 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:56,921 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T06:48:56,922 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:56,923 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened a474a06785ead13d749016bf6309b74c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70299627, jitterRate=0.04754607379436493}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T06:48:56,923 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:56,924 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for a474a06785ead13d749016bf6309b74c: Running coprocessor pre-open hook at 1731394136911Writing region info on filesystem at 1731394136911Initializing all the Stores at 1731394136912 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394136913 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394136913Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394136913Cleaning up temporary data from old regions at 1731394136921 (+8 ms)Running coprocessor post-open hooks at 1731394136923 (+2 ms)Region opened successfully at 1731394136923 2024-11-12T06:48:56,924 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., pid=96, masterSystemTime=1731394136902 2024-11-12T06:48:56,926 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:56,926 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:56,926 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=a474a06785ead13d749016bf6309b74c, regionState=OPEN, openSeqNum=5, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:56,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 because future has completed 2024-11-12T06:48:56,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-11-12T06:48:56,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 in 185 msec 2024-11-12T06:48:56,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-12T06:48:56,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a474a06785ead13d749016bf6309b74c, REOPEN/MOVE in 514 msec 2024-11-12T06:48:56,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-12T06:48:56,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 518 msec 2024-11-12T06:48:56,938 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 942 msec 2024-11-12T06:48:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-12T06:48:56,939 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:48:56,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0f1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,940 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:56,940 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:56,941 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:56,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:56,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:56,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e6acec4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:56,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:56,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:56,942 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60babc98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,943 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,943 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:56,943 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:56,943 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:56,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:56,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4485144c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:56,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:56,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:56,945 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:56,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ac64dca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,946 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:56,946 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:56,946 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:56,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4759e4f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:56,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:56,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:56,947 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:56,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5820a801, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,948 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:56,948 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:56,949 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:56,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:56,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:56,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319e0612, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:56,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:56,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:56,950 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:56,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edcb82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,951 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:56,951 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:56,951 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:56,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:56,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:56,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cefe74d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:56,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:56,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:56,952 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43ca3fbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,953 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:56,953 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:56,954 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:56,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:56,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:56,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6faa67e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:56,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:56,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:56,955 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:56,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32bcb92f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,956 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:56,956 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:56,956 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:56,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:56,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:56,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46e7bcbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:56,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:56,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:56,957 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:56,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@84ae4e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,958 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:56,958 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:56,959 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:56,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:56,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:56,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b395e5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:56,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:56,960 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2c1364, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,961 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:56,961 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:56,961 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:56,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:56,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:56,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@243519f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:56,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:56,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:56,963 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d56254, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,964 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,964 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:48:56,964 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:48:56,965 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:48:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:48:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:48:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6606bff1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:48:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:48:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:48:56,966 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:48:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43dc5b98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:48:56,967 DEBUG [Thread-1506 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:56,967 DEBUG [Thread-1512 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:56,967 DEBUG [Thread-1497 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:56,967 DEBUG [Thread-1514 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:56,967 DEBUG [Thread-1510 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:56,968 DEBUG [Thread-1508 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:56,968 DEBUG [Thread-1495 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:56,968 DEBUG [Thread-1501 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:56,969 DEBUG [Thread-1499 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:56,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:56,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:56,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:56,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:56,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:56,970 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:48:56,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:56,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:56,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:56,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:56,970 DEBUG [Thread-1503 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:48:56,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:56,971 DEBUG [Thread-1501 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:56,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:56,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:56,972 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:56,972 DEBUG [Thread-1501 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:56,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:48:56,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:48:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-12T06:48:56,973 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:56,973 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:56,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:48:56,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:48:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T06:48:56,975 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:48:56,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:48:56,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:48:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:48:56,976 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:48:56,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:48:56,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:48:56,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:48:56,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:48:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:48:56,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:48:56,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:48:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:56,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a474a06785ead13d749016bf6309b74c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T06:48:56,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=A 2024-11-12T06:48:56,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:56,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=B 2024-11-12T06:48:56,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:56,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=C 2024-11-12T06:48:56,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:57,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111251d15dfdeb0c4cd88586e9bb05a9fdaa_a474a06785ead13d749016bf6309b74c is 50, key is test_row_0/A:col10/1731394136985/Put/seqid=0 2024-11-12T06:48:57,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742153_1329 (size=12154) 2024-11-12T06:48:57,023 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:57,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731394136993, max timestamp=1731394137023Num uniques 859; 2024-11-12T06:48:57,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731394136993, max timestamp=1731394137023Num uniques 859; 2024-11-12T06:48:57,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T06:48:57,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T06:48:57,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731394136993, max timestamp=1731394137023Num uniques 859; 2024-11-12T06:48:57,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T06:48:57,033 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111251d15dfdeb0c4cd88586e9bb05a9fdaa_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111251d15dfdeb0c4cd88586e9bb05a9fdaa_a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:57,034 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/18b052f3f93c421eab78aa68c380aef8, store: [table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:48:57,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/18b052f3f93c421eab78aa68c380aef8 is 175, key is test_row_0/A:col10/1731394136985/Put/seqid=0 2024-11-12T06:48:57,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742154_1330 (size=30955) 2024-11-12T06:48:57,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394197058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:57,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394197060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:57,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:57,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:57,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394197061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:57,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:57,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394197062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:57,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:57,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394197067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T06:48:57,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T06:48:57,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:57,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:48:57,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:57,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:57,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:57,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394197177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:57,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:57,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394197187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:57,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394197187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:57,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394197188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394197188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:57,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:57,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T06:48:57,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:57,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:48:57,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:57,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:57,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T06:48:57,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394197397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:57,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:57,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394197407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:57,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394197407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:57,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394197407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:57,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394197407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:57,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T06:48:57,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:57,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:48:57,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:57,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:57,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:57,449 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/18b052f3f93c421eab78aa68c380aef8 2024-11-12T06:48:57,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/54e89077059c40089b4c5ab678765715 is 50, key is test_row_0/B:col10/1731394136985/Put/seqid=0 2024-11-12T06:48:57,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742155_1331 (size=12001) 2024-11-12T06:48:57,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T06:48:57,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:57,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:48:57,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:57,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:57,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T06:48:57,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:57,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394197707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:57,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:57,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:57,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394197727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:57,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:57,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394197727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:57,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394197727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:57,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:57,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:57,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394197727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:57,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:57,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:57,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T06:48:57,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:57,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:48:57,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:57,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:57,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:57,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/54e89077059c40089b4c5ab678765715 2024-11-12T06:48:57,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T06:48:57,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:57,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:48:57,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:57,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:57,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:57,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:57,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/630728e4b6bb48a3bcc21dbe7c0a8f85 is 50, key is test_row_0/C:col10/1731394136985/Put/seqid=0 2024-11-12T06:48:57,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742156_1332 (size=12001) 2024-11-12T06:48:58,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T06:48:58,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:58,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:48:58,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:58,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:58,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:58,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:58,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T06:48:58,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T06:48:58,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:58,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:48:58,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:58,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:58,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:58,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:48:58,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394198217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:58,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394198247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:58,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394198247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:58,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394198247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:58,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394198247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:58,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/630728e4b6bb48a3bcc21dbe7c0a8f85 2024-11-12T06:48:58,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/18b052f3f93c421eab78aa68c380aef8 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/18b052f3f93c421eab78aa68c380aef8 2024-11-12T06:48:58,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/18b052f3f93c421eab78aa68c380aef8, entries=150, sequenceid=14, filesize=30.2 K 2024-11-12T06:48:58,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/54e89077059c40089b4c5ab678765715 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/54e89077059c40089b4c5ab678765715 2024-11-12T06:48:58,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/54e89077059c40089b4c5ab678765715, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T06:48:58,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/630728e4b6bb48a3bcc21dbe7c0a8f85 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/630728e4b6bb48a3bcc21dbe7c0a8f85 2024-11-12T06:48:58,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/630728e4b6bb48a3bcc21dbe7c0a8f85, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T06:48:58,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for a474a06785ead13d749016bf6309b74c in 1325ms, sequenceid=14, compaction requested=false 2024-11-12T06:48:58,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:48:58,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T06:48:58,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:58,351 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing a474a06785ead13d749016bf6309b74c 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-12T06:48:58,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=A 2024-11-12T06:48:58,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:58,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=B 2024-11-12T06:48:58,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:58,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=C 2024-11-12T06:48:58,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:48:58,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d416c987bc484b6eac84260d6933406d_a474a06785ead13d749016bf6309b74c is 50, key is test_row_0/A:col10/1731394137066/Put/seqid=0 2024-11-12T06:48:58,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742157_1333 (size=12154) 2024-11-12T06:48:58,578 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T06:48:58,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:58,765 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d416c987bc484b6eac84260d6933406d_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d416c987bc484b6eac84260d6933406d_a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:58,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/0a01e4feb45e4ea489aa9b0ea81a1e73, store: [table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:48:58,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/0a01e4feb45e4ea489aa9b0ea81a1e73 is 175, key is test_row_0/A:col10/1731394137066/Put/seqid=0 2024-11-12T06:48:58,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742158_1334 (size=30955) 2024-11-12T06:48:59,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T06:48:59,172 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/0a01e4feb45e4ea489aa9b0ea81a1e73 2024-11-12T06:48:59,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/9096935fa6b04939acec1209ed541e40 is 50, key is test_row_0/B:col10/1731394137066/Put/seqid=0 2024-11-12T06:48:59,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742159_1335 (size=12001) 2024-11-12T06:48:59,189 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/9096935fa6b04939acec1209ed541e40 2024-11-12T06:48:59,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/798a8a30c13044e7beb88e07f7ec50fa is 50, key is test_row_0/C:col10/1731394137066/Put/seqid=0 2024-11-12T06:48:59,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742160_1336 (size=12001) 2024-11-12T06:48:59,205 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/798a8a30c13044e7beb88e07f7ec50fa 2024-11-12T06:48:59,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/0a01e4feb45e4ea489aa9b0ea81a1e73 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/0a01e4feb45e4ea489aa9b0ea81a1e73 2024-11-12T06:48:59,216 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/0a01e4feb45e4ea489aa9b0ea81a1e73, entries=150, sequenceid=49, filesize=30.2 K 2024-11-12T06:48:59,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/9096935fa6b04939acec1209ed541e40 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/9096935fa6b04939acec1209ed541e40 2024-11-12T06:48:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,220 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/9096935fa6b04939acec1209ed541e40, entries=150, sequenceid=49, filesize=11.7 K 2024-11-12T06:48:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/798a8a30c13044e7beb88e07f7ec50fa as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/798a8a30c13044e7beb88e07f7ec50fa 2024-11-12T06:48:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,224 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/798a8a30c13044e7beb88e07f7ec50fa, entries=150, sequenceid=49, filesize=11.7 K 2024-11-12T06:48:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,225 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=0 B/0 for a474a06785ead13d749016bf6309b74c in 874ms, sequenceid=49, compaction requested=false 2024-11-12T06:48:59,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:48:59,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:48:59,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-12T06:48:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-12T06:48:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-12T06:48:59,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2510 sec 2024-11-12T06:48:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.2590 sec 2024-11-12T06:48:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a474a06785ead13d749016bf6309b74c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T06:48:59,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=A 2024-11-12T06:48:59,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:59,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=B 2024-11-12T06:48:59,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=C 2024-11-12T06:48:59,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:48:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127cb4bfaec12e4c2aa106960f12694138_a474a06785ead13d749016bf6309b74c is 50, key is test_row_0/A:col10/1731394139340/Put/seqid=0 2024-11-12T06:48:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742161_1337 (size=12154) 2024-11-12T06:48:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,380 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:48:59,384 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127cb4bfaec12e4c2aa106960f12694138_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127cb4bfaec12e4c2aa106960f12694138_a474a06785ead13d749016bf6309b74c 2024-11-12T06:48:59,386 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/2e50e0731e0a415fba6bc7b8205ca50b, store: [table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:48:59,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/2e50e0731e0a415fba6bc7b8205ca50b is 175, key is test_row_0/A:col10/1731394139340/Put/seqid=0 2024-11-12T06:48:59,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742162_1338 (size=30955) 2024-11-12T06:48:59,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731394139398, max timestamp=1731394139423Num uniques 859; 2024-11-12T06:48:59,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T06:48:59,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731394139398, max timestamp=1731394139423Num uniques 859; 2024-11-12T06:48:59,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731394139398, max timestamp=1731394139423Num uniques 859; 2024-11-12T06:48:59,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T06:48:59,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T06:48:59,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394199449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:59,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394199455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:59,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394199459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:59,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394199469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:59,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:59,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394199469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:59,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:59,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394199577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:59,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:59,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394199577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:59,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:59,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394199577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:59,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:59,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:59,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394199586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:59,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394199587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:59,800 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=59, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/2e50e0731e0a415fba6bc7b8205ca50b 2024-11-12T06:48:59,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/e8113fac2fa94f9eac85a373b797fde3 is 50, key is test_row_0/B:col10/1731394139340/Put/seqid=0 2024-11-12T06:48:59,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394199797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:59,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:59,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394199797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:59,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394199797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:59,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394199797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:48:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394199798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:48:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:48:59,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:48:59,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742163_1339 (size=12001) 2024-11-12T06:49:00,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394200117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:00,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394200117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:00,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394200117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:00,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394200117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:00,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394200117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:00,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/e8113fac2fa94f9eac85a373b797fde3 2024-11-12T06:49:00,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/4953e493bd5b47ceb46fd651f5a47a45 is 50, key is test_row_0/C:col10/1731394139340/Put/seqid=0 2024-11-12T06:49:00,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742164_1340 (size=12001) 2024-11-12T06:49:00,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/4953e493bd5b47ceb46fd651f5a47a45 2024-11-12T06:49:00,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/2e50e0731e0a415fba6bc7b8205ca50b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/2e50e0731e0a415fba6bc7b8205ca50b 2024-11-12T06:49:00,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/2e50e0731e0a415fba6bc7b8205ca50b, entries=150, sequenceid=59, filesize=30.2 K 2024-11-12T06:49:00,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/e8113fac2fa94f9eac85a373b797fde3 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/e8113fac2fa94f9eac85a373b797fde3 2024-11-12T06:49:00,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/e8113fac2fa94f9eac85a373b797fde3, entries=150, sequenceid=59, filesize=11.7 K 2024-11-12T06:49:00,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/4953e493bd5b47ceb46fd651f5a47a45 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/4953e493bd5b47ceb46fd651f5a47a45 2024-11-12T06:49:00,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/4953e493bd5b47ceb46fd651f5a47a45, entries=150, sequenceid=59, filesize=11.7 K 2024-11-12T06:49:00,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for a474a06785ead13d749016bf6309b74c in 1273ms, sequenceid=59, compaction requested=true 2024-11-12T06:49:00,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:00,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a474a06785ead13d749016bf6309b74c:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:49:00,639 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:00,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:00,639 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:00,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a474a06785ead13d749016bf6309b74c:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:49:00,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:00,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a474a06785ead13d749016bf6309b74c:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:49:00,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:00,640 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:00,640 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:00,640 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): a474a06785ead13d749016bf6309b74c/A is initiating minor compaction (all files) 2024-11-12T06:49:00,640 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): a474a06785ead13d749016bf6309b74c/B is initiating minor compaction (all files) 2024-11-12T06:49:00,640 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a474a06785ead13d749016bf6309b74c/A in TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:00,640 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a474a06785ead13d749016bf6309b74c/B in TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:00,640 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/18b052f3f93c421eab78aa68c380aef8, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/0a01e4feb45e4ea489aa9b0ea81a1e73, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/2e50e0731e0a415fba6bc7b8205ca50b] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp, totalSize=90.7 K 2024-11-12T06:49:00,640 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/54e89077059c40089b4c5ab678765715, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/9096935fa6b04939acec1209ed541e40, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/e8113fac2fa94f9eac85a373b797fde3] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp, totalSize=35.2 K 2024-11-12T06:49:00,640 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:00,640 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. files: [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/18b052f3f93c421eab78aa68c380aef8, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/0a01e4feb45e4ea489aa9b0ea81a1e73, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/2e50e0731e0a415fba6bc7b8205ca50b] 2024-11-12T06:49:00,641 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 54e89077059c40089b4c5ab678765715, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731394136982 2024-11-12T06:49:00,641 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18b052f3f93c421eab78aa68c380aef8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731394136982 2024-11-12T06:49:00,641 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a01e4feb45e4ea489aa9b0ea81a1e73, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731394137054 2024-11-12T06:49:00,642 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 9096935fa6b04939acec1209ed541e40, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731394137054 2024-11-12T06:49:00,643 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e50e0731e0a415fba6bc7b8205ca50b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731394139339 2024-11-12T06:49:00,643 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting e8113fac2fa94f9eac85a373b797fde3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731394139339 2024-11-12T06:49:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:00,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a474a06785ead13d749016bf6309b74c 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-11-12T06:49:00,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=A 2024-11-12T06:49:00,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:00,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=B 2024-11-12T06:49:00,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:00,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=C 2024-11-12T06:49:00,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:00,651 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a474a06785ead13d749016bf6309b74c#B#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:00,651 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/ba132d04fdef41b9a7391048b54f9968 is 50, key is test_row_0/B:col10/1731394139340/Put/seqid=0 2024-11-12T06:49:00,651 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:00,654 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112880ba51b8bb549c1afadf1dbb4b1cddd_a474a06785ead13d749016bf6309b74c store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:00,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742165_1341 (size=12104) 2024-11-12T06:49:00,656 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112880ba51b8bb549c1afadf1dbb4b1cddd_a474a06785ead13d749016bf6309b74c, store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:00,656 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112880ba51b8bb549c1afadf1dbb4b1cddd_a474a06785ead13d749016bf6309b74c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:00,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111232610c2b1a1b4a04b7b868870153a552_a474a06785ead13d749016bf6309b74c is 50, key is test_row_0/A:col10/1731394139467/Put/seqid=0 2024-11-12T06:49:00,662 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/ba132d04fdef41b9a7391048b54f9968 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/ba132d04fdef41b9a7391048b54f9968 2024-11-12T06:49:00,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742166_1342 (size=4469) 2024-11-12T06:49:00,666 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a474a06785ead13d749016bf6309b74c/B of a474a06785ead13d749016bf6309b74c into ba132d04fdef41b9a7391048b54f9968(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:00,666 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:00,666 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., storeName=a474a06785ead13d749016bf6309b74c/B, priority=13, startTime=1731394140639; duration=0sec 2024-11-12T06:49:00,667 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:00,667 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a474a06785ead13d749016bf6309b74c:B 2024-11-12T06:49:00,667 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:00,668 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:00,668 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): a474a06785ead13d749016bf6309b74c/C is initiating minor compaction (all files) 2024-11-12T06:49:00,668 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a474a06785ead13d749016bf6309b74c/C in TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:00,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742167_1343 (size=14594) 2024-11-12T06:49:00,668 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/630728e4b6bb48a3bcc21dbe7c0a8f85, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/798a8a30c13044e7beb88e07f7ec50fa, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/4953e493bd5b47ceb46fd651f5a47a45] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp, totalSize=35.2 K 2024-11-12T06:49:00,669 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:00,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394200648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:00,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:00,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394200652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:00,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:00,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394200653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:00,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394200653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394200656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:00,674 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111232610c2b1a1b4a04b7b868870153a552_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111232610c2b1a1b4a04b7b868870153a552_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:00,674 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 630728e4b6bb48a3bcc21dbe7c0a8f85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731394136982 2024-11-12T06:49:00,675 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 798a8a30c13044e7beb88e07f7ec50fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731394137054 2024-11-12T06:49:00,675 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/88eeb0011a7c476c8ccd581b1c899686, store: [table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:00,675 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 4953e493bd5b47ceb46fd651f5a47a45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731394139339 2024-11-12T06:49:00,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/88eeb0011a7c476c8ccd581b1c899686 is 175, key is test_row_0/A:col10/1731394139467/Put/seqid=0 2024-11-12T06:49:00,684 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a474a06785ead13d749016bf6309b74c#C#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:00,684 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/7ec464ff4b0a4315a07bd4b7831ade88 is 50, key is test_row_0/C:col10/1731394139340/Put/seqid=0 2024-11-12T06:49:00,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742168_1344 (size=39549) 2024-11-12T06:49:00,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742169_1345 (size=12104) 2024-11-12T06:49:00,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394200777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394200777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:00,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394200777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394200777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:00,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394200777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:00,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394200987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394200997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394200998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394200998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394200998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,064 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a474a06785ead13d749016bf6309b74c#A#compaction#286 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:01,065 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/e60d9b3a27de430ebc47984a88234226 is 175, key is test_row_0/A:col10/1731394139340/Put/seqid=0 2024-11-12T06:49:01,087 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/88eeb0011a7c476c8ccd581b1c899686 2024-11-12T06:49:01,102 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/7ec464ff4b0a4315a07bd4b7831ade88 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/7ec464ff4b0a4315a07bd4b7831ade88 2024-11-12T06:49:01,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742170_1346 (size=31058) 2024-11-12T06:49:01,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/405ee515f9284c4e816eb670ba6273cf is 50, key is test_row_0/B:col10/1731394139467/Put/seqid=0 2024-11-12T06:49:01,108 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a474a06785ead13d749016bf6309b74c/C of a474a06785ead13d749016bf6309b74c into 7ec464ff4b0a4315a07bd4b7831ade88(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:01,109 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:01,109 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., storeName=a474a06785ead13d749016bf6309b74c/C, priority=13, startTime=1731394140639; duration=0sec 2024-11-12T06:49:01,109 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:01,109 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a474a06785ead13d749016bf6309b74c:C 2024-11-12T06:49:01,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742171_1347 (size=12001) 2024-11-12T06:49:01,112 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/e60d9b3a27de430ebc47984a88234226 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/e60d9b3a27de430ebc47984a88234226 2024-11-12T06:49:01,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/405ee515f9284c4e816eb670ba6273cf 2024-11-12T06:49:01,120 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a474a06785ead13d749016bf6309b74c/A of a474a06785ead13d749016bf6309b74c into e60d9b3a27de430ebc47984a88234226(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:01,120 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:01,121 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., storeName=a474a06785ead13d749016bf6309b74c/A, priority=13, startTime=1731394140639; duration=0sec 2024-11-12T06:49:01,121 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:01,121 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a474a06785ead13d749016bf6309b74c:A 2024-11-12T06:49:01,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/6482cb4b6d5d4dc4b29ac9094fb43688 is 50, key is test_row_0/C:col10/1731394139467/Put/seqid=0 2024-11-12T06:49:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T06:49:01,138 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:49:01,138 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:49:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-12T06:49:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T06:49:01,141 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:49:01,142 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:49:01,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:49:01,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742172_1348 (size=12001) 2024-11-12T06:49:01,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/6482cb4b6d5d4dc4b29ac9094fb43688 2024-11-12T06:49:01,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/88eeb0011a7c476c8ccd581b1c899686 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/88eeb0011a7c476c8ccd581b1c899686 2024-11-12T06:49:01,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/88eeb0011a7c476c8ccd581b1c899686, entries=200, sequenceid=96, filesize=38.6 K 2024-11-12T06:49:01,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/405ee515f9284c4e816eb670ba6273cf as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/405ee515f9284c4e816eb670ba6273cf 2024-11-12T06:49:01,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/405ee515f9284c4e816eb670ba6273cf, entries=150, sequenceid=96, filesize=11.7 K 2024-11-12T06:49:01,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/6482cb4b6d5d4dc4b29ac9094fb43688 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/6482cb4b6d5d4dc4b29ac9094fb43688 2024-11-12T06:49:01,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/6482cb4b6d5d4dc4b29ac9094fb43688, entries=150, sequenceid=96, filesize=11.7 K 2024-11-12T06:49:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for a474a06785ead13d749016bf6309b74c in 520ms, sequenceid=96, compaction requested=false 2024-11-12T06:49:01,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T06:49:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T06:49:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,294 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing a474a06785ead13d749016bf6309b74c 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-12T06:49:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=A 2024-11-12T06:49:01,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:01,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=B 2024-11-12T06:49:01,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:01,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=C 2024-11-12T06:49:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124a19ccb20d1a4b27a60f74d890de79a8_a474a06785ead13d749016bf6309b74c is 50, key is test_row_1/A:col10/1731394140654/Put/seqid=0 2024-11-12T06:49:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:01,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742173_1349 (size=9714) 2024-11-12T06:49:01,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,337 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124a19ccb20d1a4b27a60f74d890de79a8_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124a19ccb20d1a4b27a60f74d890de79a8_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:01,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/81e4b73c837b4d668064add03a47fb66, store: [table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:01,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/81e4b73c837b4d668064add03a47fb66 is 175, key is test_row_1/A:col10/1731394140654/Put/seqid=0 2024-11-12T06:49:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742174_1350 (size=22361) 2024-11-12T06:49:01,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,344 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=107, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/81e4b73c837b4d668064add03a47fb66 2024-11-12T06:49:01,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:01,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/2f2a2e16b9354a1c8c96bfc29663e27c is 50, key is test_row_1/B:col10/1731394140654/Put/seqid=0 2024-11-12T06:49:01,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742175_1351 (size=9657) 2024-11-12T06:49:01,385 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/2f2a2e16b9354a1c8c96bfc29663e27c 2024-11-12T06:49:01,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394141361, max timestamp=1731394141391Num uniques 859; 2024-11-12T06:49:01,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394141361, max timestamp=1731394141391Num uniques 859; 2024-11-12T06:49:01,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394141361, max timestamp=1731394141391Num uniques 859; 2024-11-12T06:49:01,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:49:01,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:49:01,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:49:01,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/3d2060d290de440091ae84dec9f5c3a9 is 50, key is test_row_1/C:col10/1731394140654/Put/seqid=0 2024-11-12T06:49:01,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742176_1352 (size=9657) 2024-11-12T06:49:01,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731394141417, max timestamp=1731394141445Num uniques 1018; 2024-11-12T06:49:01,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731394141417, max timestamp=1731394141445Num uniques 1018; 2024-11-12T06:49:01,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-12T06:49:01,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-12T06:49:01,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731394141417, max timestamp=1731394141445Num uniques 1018; 2024-11-12T06:49:01,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-12T06:49:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T06:49:01,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394201475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394201476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394201478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394201484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394201485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394201597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394201597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394201597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394201597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394201607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T06:49:01,805 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/3d2060d290de440091ae84dec9f5c3a9 2024-11-12T06:49:01,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/81e4b73c837b4d668064add03a47fb66 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/81e4b73c837b4d668064add03a47fb66 2024-11-12T06:49:01,812 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/81e4b73c837b4d668064add03a47fb66, entries=100, sequenceid=107, filesize=21.8 K 2024-11-12T06:49:01,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/2f2a2e16b9354a1c8c96bfc29663e27c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/2f2a2e16b9354a1c8c96bfc29663e27c 2024-11-12T06:49:01,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394201807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,816 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/2f2a2e16b9354a1c8c96bfc29663e27c, entries=100, sequenceid=107, filesize=9.4 K 2024-11-12T06:49:01,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/3d2060d290de440091ae84dec9f5c3a9 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/3d2060d290de440091ae84dec9f5c3a9 2024-11-12T06:49:01,820 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/3d2060d290de440091ae84dec9f5c3a9, entries=100, sequenceid=107, filesize=9.4 K 2024-11-12T06:49:01,821 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for a474a06785ead13d749016bf6309b74c in 527ms, sequenceid=107, compaction requested=true 2024-11-12T06:49:01,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:01,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:01,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-12T06:49:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-12T06:49:01,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-12T06:49:01,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 680 msec 2024-11-12T06:49:01,825 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 686 msec 2024-11-12T06:49:01,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:01,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a474a06785ead13d749016bf6309b74c 3/3 column families, dataSize=161.02 KB heapSize=450.47 KB 2024-11-12T06:49:01,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=A 2024-11-12T06:49:01,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:49:01,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=B 2024-11-12T06:49:01,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:49:01,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=C 2024-11-12T06:49:01,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:49:01,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124eb67913be504945993455a4b820b87a_a474a06785ead13d749016bf6309b74c is 50, key is test_row_0/A:col10/1731394141462/Put/seqid=0 2024-11-12T06:49:01,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742177_1353 (size=14794) 2024-11-12T06:49:01,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394201829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394201830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394201835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394201842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394201947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394201947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394201947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:01,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394201957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:02,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394202127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:02,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394202167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:02,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:02,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:02,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:02,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394202167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:02,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394202167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:02,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394202167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:02,238 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:02,241 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124eb67913be504945993455a4b820b87a_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124eb67913be504945993455a4b820b87a_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:02,241 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/878ff37d9f1445a38da969c98049eeef, store: [table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:02,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/878ff37d9f1445a38da969c98049eeef is 175, key is test_row_0/A:col10/1731394141462/Put/seqid=0 2024-11-12T06:49:02,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742178_1354 (size=39749) 2024-11-12T06:49:02,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T06:49:02,277 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:49:02,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:49:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-12T06:49:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T06:49:02,279 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:49:02,280 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:49:02,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:49:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T06:49:02,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T06:49:02,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:02,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:02,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:02,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:02,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:02,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:02,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:02,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394202477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:02,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:02,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:02,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:02,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:02,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394202487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:02,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:02,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394202487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:02,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:02,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394202487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:02,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T06:49:02,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:02,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:02,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:02,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:02,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:02,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T06:49:02,645 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=162, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/878ff37d9f1445a38da969c98049eeef 2024-11-12T06:49:02,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/48b246fc4b77443f8e5bad1b08e84829 is 50, key is test_row_0/B:col10/1731394141462/Put/seqid=0 2024-11-12T06:49:02,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:02,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394202647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:02,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:02,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:02,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:02,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742179_1355 (size=12151) 2024-11-12T06:49:02,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T06:49:02,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:02,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:02,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:02,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:02,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:02,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:02,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T06:49:02,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:02,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:02,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:02,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:02,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:02,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T06:49:03,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394202997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:03,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394203007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:03,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:03,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394203007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:03,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:03,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:03,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:03,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394203007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:03,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:03,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:03,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:03,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:03,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:03,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T06:49:03,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:03,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:03,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:03,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:03,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:03,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/48b246fc4b77443f8e5bad1b08e84829 2024-11-12T06:49:03,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/62e7696abf994f41a8b5612fb4f03218 is 50, key is test_row_0/C:col10/1731394141462/Put/seqid=0 2024-11-12T06:49:03,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742180_1356 (size=12151) 2024-11-12T06:49:03,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T06:49:03,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:03,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:03,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:03,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:03,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:03,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T06:49:03,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:03,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:03,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:03,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:03,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:03,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T06:49:03,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/62e7696abf994f41a8b5612fb4f03218 2024-11-12T06:49:03,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/878ff37d9f1445a38da969c98049eeef as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/878ff37d9f1445a38da969c98049eeef 2024-11-12T06:49:03,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/878ff37d9f1445a38da969c98049eeef, entries=200, sequenceid=162, filesize=38.8 K 2024-11-12T06:49:03,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/48b246fc4b77443f8e5bad1b08e84829 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/48b246fc4b77443f8e5bad1b08e84829 2024-11-12T06:49:03,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/48b246fc4b77443f8e5bad1b08e84829, entries=150, sequenceid=162, filesize=11.9 K 2024-11-12T06:49:03,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/62e7696abf994f41a8b5612fb4f03218 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/62e7696abf994f41a8b5612fb4f03218 2024-11-12T06:49:03,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/62e7696abf994f41a8b5612fb4f03218, entries=150, sequenceid=162, filesize=11.9 K 2024-11-12T06:49:03,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~450.42 KB/461232, currentSize=26.84 KB/27480 for a474a06785ead13d749016bf6309b74c in 1662ms, sequenceid=162, compaction requested=true 2024-11-12T06:49:03,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:03,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a474a06785ead13d749016bf6309b74c:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:49:03,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:03,490 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:49:03,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a474a06785ead13d749016bf6309b74c:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:49:03,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:03,490 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:49:03,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a474a06785ead13d749016bf6309b74c:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:49:03,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:03,491 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132717 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:49:03,491 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): a474a06785ead13d749016bf6309b74c/A is initiating minor compaction (all files) 2024-11-12T06:49:03,491 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:49:03,491 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): a474a06785ead13d749016bf6309b74c/B is initiating minor compaction (all files) 2024-11-12T06:49:03,491 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a474a06785ead13d749016bf6309b74c/A in TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:03,491 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a474a06785ead13d749016bf6309b74c/B in TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:03,491 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/e60d9b3a27de430ebc47984a88234226, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/88eeb0011a7c476c8ccd581b1c899686, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/81e4b73c837b4d668064add03a47fb66, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/878ff37d9f1445a38da969c98049eeef] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp, totalSize=129.6 K 2024-11-12T06:49:03,491 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/ba132d04fdef41b9a7391048b54f9968, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/405ee515f9284c4e816eb670ba6273cf, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/2f2a2e16b9354a1c8c96bfc29663e27c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/48b246fc4b77443f8e5bad1b08e84829] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp, totalSize=44.8 K 2024-11-12T06:49:03,491 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:03,491 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. files: [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/e60d9b3a27de430ebc47984a88234226, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/88eeb0011a7c476c8ccd581b1c899686, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/81e4b73c837b4d668064add03a47fb66, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/878ff37d9f1445a38da969c98049eeef] 2024-11-12T06:49:03,491 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting e60d9b3a27de430ebc47984a88234226, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731394139339 2024-11-12T06:49:03,491 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting ba132d04fdef41b9a7391048b54f9968, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731394139339 2024-11-12T06:49:03,492 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88eeb0011a7c476c8ccd581b1c899686, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731394139453 2024-11-12T06:49:03,492 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 405ee515f9284c4e816eb670ba6273cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731394139467 2024-11-12T06:49:03,492 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f2a2e16b9354a1c8c96bfc29663e27c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731394140650 2024-11-12T06:49:03,492 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81e4b73c837b4d668064add03a47fb66, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731394140650 2024-11-12T06:49:03,492 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 48b246fc4b77443f8e5bad1b08e84829, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731394141462 2024-11-12T06:49:03,493 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 878ff37d9f1445a38da969c98049eeef, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731394141462 2024-11-12T06:49:03,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T06:49:03,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:03,500 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing a474a06785ead13d749016bf6309b74c 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-12T06:49:03,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=A 2024-11-12T06:49:03,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:03,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=B 2024-11-12T06:49:03,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:03,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=C 2024-11-12T06:49:03,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:03,503 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a474a06785ead13d749016bf6309b74c#B#compaction#297 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:03,504 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/81b1223329784193b6a121330f6cade6 is 50, key is test_row_0/B:col10/1731394141462/Put/seqid=0 2024-11-12T06:49:03,507 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:03,515 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112cbbe3b150f744bf9a63e53838b6d1e8a_a474a06785ead13d749016bf6309b74c store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:03,517 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112cbbe3b150f744bf9a63e53838b6d1e8a_a474a06785ead13d749016bf6309b74c, store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:03,518 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112cbbe3b150f744bf9a63e53838b6d1e8a_a474a06785ead13d749016bf6309b74c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:03,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742181_1357 (size=12391) 2024-11-12T06:49:03,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127b3cb020cf2c444bb3e2f5e559aa25b1_a474a06785ead13d749016bf6309b74c is 50, key is test_row_0/A:col10/1731394141833/Put/seqid=0 2024-11-12T06:49:03,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742182_1358 (size=12304) 2024-11-12T06:49:03,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742183_1359 (size=4469) 2024-11-12T06:49:03,547 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a474a06785ead13d749016bf6309b74c#A#compaction#298 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:03,547 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/07d946caabd1486c94ba17c104c3bf0c is 175, key is test_row_0/A:col10/1731394141462/Put/seqid=0 2024-11-12T06:49:03,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742184_1360 (size=31345) 2024-11-12T06:49:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:03,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:03,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394143728, max timestamp=1731394143817Num uniques 859; 2024-11-12T06:49:03,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394143728, max timestamp=1731394143817Num uniques 859; 2024-11-12T06:49:03,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:03,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:03,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394143728, max timestamp=1731394143817Num uniques 859; 2024-11-12T06:49:03,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:03,928 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/81b1223329784193b6a121330f6cade6 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/81b1223329784193b6a121330f6cade6 2024-11-12T06:49:03,932 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a474a06785ead13d749016bf6309b74c/B of a474a06785ead13d749016bf6309b74c into 81b1223329784193b6a121330f6cade6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:03,932 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:03,932 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., storeName=a474a06785ead13d749016bf6309b74c/B, priority=12, startTime=1731394143490; duration=0sec 2024-11-12T06:49:03,932 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:03,932 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a474a06785ead13d749016bf6309b74c:B 2024-11-12T06:49:03,932 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:49:03,933 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:49:03,933 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): a474a06785ead13d749016bf6309b74c/C is initiating minor compaction (all files) 2024-11-12T06:49:03,933 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a474a06785ead13d749016bf6309b74c/C in TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:03,933 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/7ec464ff4b0a4315a07bd4b7831ade88, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/6482cb4b6d5d4dc4b29ac9094fb43688, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/3d2060d290de440091ae84dec9f5c3a9, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/62e7696abf994f41a8b5612fb4f03218] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp, totalSize=44.8 K 2024-11-12T06:49:03,933 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ec464ff4b0a4315a07bd4b7831ade88, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731394139339 2024-11-12T06:49:03,934 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 6482cb4b6d5d4dc4b29ac9094fb43688, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731394139467 2024-11-12T06:49:03,934 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d2060d290de440091ae84dec9f5c3a9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731394140650 2024-11-12T06:49:03,934 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 62e7696abf994f41a8b5612fb4f03218, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731394141462 2024-11-12T06:49:03,943 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a474a06785ead13d749016bf6309b74c#C#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:03,943 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/6fdabf840350465280294d393a67f0bb is 50, key is test_row_0/C:col10/1731394141462/Put/seqid=0 2024-11-12T06:49:03,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:03,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742185_1361 (size=12391) 2024-11-12T06:49:03,948 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127b3cb020cf2c444bb3e2f5e559aa25b1_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127b3cb020cf2c444bb3e2f5e559aa25b1_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:03,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/1b4180e45ad7493380ebc350a12663a7, store: [table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:03,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/1b4180e45ad7493380ebc350a12663a7 is 175, key is test_row_0/A:col10/1731394141833/Put/seqid=0 2024-11-12T06:49:03,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742186_1362 (size=31105) 2024-11-12T06:49:03,957 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/07d946caabd1486c94ba17c104c3bf0c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/07d946caabd1486c94ba17c104c3bf0c 2024-11-12T06:49:03,961 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a474a06785ead13d749016bf6309b74c/A of a474a06785ead13d749016bf6309b74c into 07d946caabd1486c94ba17c104c3bf0c(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:03,961 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:03,961 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., storeName=a474a06785ead13d749016bf6309b74c/A, priority=12, startTime=1731394143490; duration=0sec 2024-11-12T06:49:03,961 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:03,961 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a474a06785ead13d749016bf6309b74c:A 2024-11-12T06:49:04,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731394143966, max timestamp=1731394144007Num uniques 1068; 2024-11-12T06:49:04,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731394143966, max timestamp=1731394144007Num uniques 1068; 2024-11-12T06:49:04,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-12T06:49:04,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-12T06:49:04,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731394143966, max timestamp=1731394144007Num uniques 1068; 2024-11-12T06:49:04,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-12T06:49:04,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731394144071, max timestamp=1731394144103Num uniques 1027; 2024-11-12T06:49:04,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731394144071, max timestamp=1731394144103Num uniques 1027; 2024-11-12T06:49:04,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731394144071, max timestamp=1731394144103Num uniques 1027; 2024-11-12T06:49:04,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-12T06:49:04,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-12T06:49:04,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-12T06:49:04,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731394144124, max timestamp=1731394144151Num uniques 1036; 2024-11-12T06:49:04,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=15386, suffix heap size=135712,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=336 2024-11-12T06:49:04,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731394144124, max timestamp=1731394144151Num uniques 1036; 2024-11-12T06:49:04,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=15386, suffix heap size=135712,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=336 2024-11-12T06:49:04,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731394144124, max timestamp=1731394144151Num uniques 1036; 2024-11-12T06:49:04,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=15386, suffix heap size=135712,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=336 2024-11-12T06:49:04,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:04,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394204182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:04,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:04,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394204189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:04,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:04,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731394144179, max timestamp=1731394144192Num uniques 1195; 2024-11-12T06:49:04,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=15800, suffix heap size=152248,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=345 2024-11-12T06:49:04,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731394144179, max timestamp=1731394144192Num uniques 1195; 2024-11-12T06:49:04,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=15800, suffix heap size=152248,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=345 2024-11-12T06:49:04,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731394144179, max timestamp=1731394144192Num uniques 1195; 2024-11-12T06:49:04,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:04,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=15800, suffix heap size=152248,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=345 2024-11-12T06:49:04,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394204250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394204252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:04,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:04,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:04,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394204255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:04,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:04,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:04,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394204307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:04,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:04,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:04,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394204308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:04,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:04,349 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/6fdabf840350465280294d393a67f0bb as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/6fdabf840350465280294d393a67f0bb 2024-11-12T06:49:04,352 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/1b4180e45ad7493380ebc350a12663a7 2024-11-12T06:49:04,353 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a474a06785ead13d749016bf6309b74c/C of a474a06785ead13d749016bf6309b74c into 6fdabf840350465280294d393a67f0bb(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:04,353 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:04,353 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., storeName=a474a06785ead13d749016bf6309b74c/C, priority=12, startTime=1731394143490; duration=0sec 2024-11-12T06:49:04,353 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:04,353 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a474a06785ead13d749016bf6309b74c:C 2024-11-12T06:49:04,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/f7be1a2ba67149c39f0d87d5c7bf0090 is 50, key is test_row_0/B:col10/1731394141833/Put/seqid=0 2024-11-12T06:49:04,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742187_1363 (size=12151) 2024-11-12T06:49:04,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:04,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394204366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:04,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:04,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:04,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394204367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:04,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:04,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:04,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394204377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:04,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T06:49:04,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:04,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394204527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:04,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:04,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394204527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:04,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:04,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394204587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:04,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394204587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394204587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:04,761 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/f7be1a2ba67149c39f0d87d5c7bf0090 2024-11-12T06:49:04,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/6e3709f4cb92459bacfe2fc42b303f1b is 50, key is test_row_0/C:col10/1731394141833/Put/seqid=0 2024-11-12T06:49:04,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742188_1364 (size=12151) 2024-11-12T06:49:04,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:04,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394204837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:04,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:04,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394204837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:04,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:04,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:04,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:04,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394204897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:04,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394204897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:04,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394204897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:04,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:05,170 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/6e3709f4cb92459bacfe2fc42b303f1b 2024-11-12T06:49:05,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/1b4180e45ad7493380ebc350a12663a7 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/1b4180e45ad7493380ebc350a12663a7 2024-11-12T06:49:05,177 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/1b4180e45ad7493380ebc350a12663a7, entries=150, sequenceid=169, filesize=30.4 K 2024-11-12T06:49:05,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/f7be1a2ba67149c39f0d87d5c7bf0090 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/f7be1a2ba67149c39f0d87d5c7bf0090 2024-11-12T06:49:05,180 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/f7be1a2ba67149c39f0d87d5c7bf0090, entries=150, sequenceid=169, filesize=11.9 K 2024-11-12T06:49:05,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/6e3709f4cb92459bacfe2fc42b303f1b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/6e3709f4cb92459bacfe2fc42b303f1b 2024-11-12T06:49:05,185 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/6e3709f4cb92459bacfe2fc42b303f1b, entries=150, sequenceid=169, filesize=11.9 K 2024-11-12T06:49:05,186 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=161.02 KB/164880 for a474a06785ead13d749016bf6309b74c in 1686ms, sequenceid=169, compaction requested=false 2024-11-12T06:49:05,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:05,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:05,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-12T06:49:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-12T06:49:05,189 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-12T06:49:05,189 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9070 sec 2024-11-12T06:49:05,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.9110 sec 2024-11-12T06:49:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:05,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a474a06785ead13d749016bf6309b74c 3/3 column families, dataSize=167.72 KB heapSize=467.86 KB 2024-11-12T06:49:05,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=A 2024-11-12T06:49:05,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:49:05,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=B 2024-11-12T06:49:05,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:49:05,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=C 2024-11-12T06:49:05,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:49:05,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111237a8963f9fea4713be57120239cbc237_a474a06785ead13d749016bf6309b74c is 50, key is test_row_0/A:col10/1731394145363/Put/seqid=0 2024-11-12T06:49:05,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742189_1365 (size=14994) 2024-11-12T06:49:05,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394205374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:05,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:05,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:05,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:05,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394205376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:05,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:05,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:05,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:05,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394205417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:05,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:05,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:05,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394205417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:05,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:05,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:05,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:05,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:05,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394205417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:05,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:05,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:05,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:05,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:05,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394205487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:05,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:05,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394205487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:05,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:05,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:05,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:05,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:05,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:05,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:05,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394205707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:05,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394205707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:05,776 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:05,779 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111237a8963f9fea4713be57120239cbc237_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111237a8963f9fea4713be57120239cbc237_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:05,780 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/5ac5646bd58f44b2bd96d36627e32694, store: [table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:05,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/5ac5646bd58f44b2bd96d36627e32694 is 175, key is test_row_0/A:col10/1731394145363/Put/seqid=0 2024-11-12T06:49:05,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742190_1366 (size=39949) 2024-11-12T06:49:06,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394206017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:06,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:06,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:06,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394206017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:06,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:06,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:06,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:06,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:06,184 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/5ac5646bd58f44b2bd96d36627e32694 2024-11-12T06:49:06,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/e78f020042dd4e82b2c108cacc9825bf is 50, key is test_row_0/B:col10/1731394145363/Put/seqid=0 2024-11-12T06:49:06,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742191_1367 (size=12301) 2024-11-12T06:49:06,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394206427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:06,435 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_1' in a474a06785ead13d749016bf6309b74c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:49:06,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:06,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:06,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:06,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394206437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:06,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394206437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T06:49:06,447 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_1' in a474a06785ead13d749016bf6309b74c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:49:06,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:06,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:06,447 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_0' in a474a06785ead13d749016bf6309b74c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:49:06,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:06,448 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:49:06,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:06,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:06,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:06,448 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:49:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-12T06:49:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T06:49:06,449 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:49:06,450 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:49:06,450 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:49:06,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:06,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394206527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:06,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:06,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:06,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:06,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394206537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:06,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:06,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:06,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T06:49:06,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-12T06:49:06,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:06,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:06,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:06,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:06,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:06,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:06,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/e78f020042dd4e82b2c108cacc9825bf 2024-11-12T06:49:06,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/9ecbef4dfa2e4ebe80982c26b92eb7e6 is 50, key is test_row_0/C:col10/1731394145363/Put/seqid=0 2024-11-12T06:49:06,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742192_1368 (size=12301) 2024-11-12T06:49:06,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-12T06:49:06,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:06,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:06,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:06,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:06,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:06,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T06:49:06,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-12T06:49:06,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:06,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:06,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:06,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:06,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:07,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/9ecbef4dfa2e4ebe80982c26b92eb7e6 2024-11-12T06:49:07,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/5ac5646bd58f44b2bd96d36627e32694 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/5ac5646bd58f44b2bd96d36627e32694 2024-11-12T06:49:07,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/5ac5646bd58f44b2bd96d36627e32694, entries=200, sequenceid=279, filesize=39.0 K 2024-11-12T06:49:07,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/e78f020042dd4e82b2c108cacc9825bf as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/e78f020042dd4e82b2c108cacc9825bf 2024-11-12T06:49:07,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/e78f020042dd4e82b2c108cacc9825bf, entries=150, sequenceid=279, filesize=12.0 K 2024-11-12T06:49:07,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/9ecbef4dfa2e4ebe80982c26b92eb7e6 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/9ecbef4dfa2e4ebe80982c26b92eb7e6 2024-11-12T06:49:07,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/9ecbef4dfa2e4ebe80982c26b92eb7e6, entries=150, sequenceid=279, filesize=12.0 K 2024-11-12T06:49:07,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~467.81 KB/479040, currentSize=20.13 KB/20610 for a474a06785ead13d749016bf6309b74c in 1693ms, sequenceid=279, compaction requested=true 2024-11-12T06:49:07,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:07,058 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:07,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a474a06785ead13d749016bf6309b74c:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:49:07,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:07,058 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:07,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a474a06785ead13d749016bf6309b74c:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:49:07,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:07,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a474a06785ead13d749016bf6309b74c:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:49:07,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:07,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-12T06:49:07,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:07,060 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing a474a06785ead13d749016bf6309b74c 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-12T06:49:07,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=A 2024-11-12T06:49:07,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:07,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=B 2024-11-12T06:49:07,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:07,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=C 2024-11-12T06:49:07,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:07,062 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:07,062 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:07,062 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): a474a06785ead13d749016bf6309b74c/B is initiating minor compaction (all files) 2024-11-12T06:49:07,062 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): a474a06785ead13d749016bf6309b74c/A is initiating minor compaction (all files) 2024-11-12T06:49:07,062 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a474a06785ead13d749016bf6309b74c/B in TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:07,062 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a474a06785ead13d749016bf6309b74c/A in TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:07,062 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/81b1223329784193b6a121330f6cade6, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/f7be1a2ba67149c39f0d87d5c7bf0090, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/e78f020042dd4e82b2c108cacc9825bf] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp, totalSize=36.0 K 2024-11-12T06:49:07,062 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/07d946caabd1486c94ba17c104c3bf0c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/1b4180e45ad7493380ebc350a12663a7, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/5ac5646bd58f44b2bd96d36627e32694] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp, totalSize=100.0 K 2024-11-12T06:49:07,062 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:07,062 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. files: [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/07d946caabd1486c94ba17c104c3bf0c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/1b4180e45ad7493380ebc350a12663a7, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/5ac5646bd58f44b2bd96d36627e32694] 2024-11-12T06:49:07,063 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 81b1223329784193b6a121330f6cade6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731394141462 2024-11-12T06:49:07,063 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07d946caabd1486c94ba17c104c3bf0c, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731394141462 2024-11-12T06:49:07,063 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting f7be1a2ba67149c39f0d87d5c7bf0090, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731394141827 2024-11-12T06:49:07,063 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting e78f020042dd4e82b2c108cacc9825bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731394144227 2024-11-12T06:49:07,064 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b4180e45ad7493380ebc350a12663a7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731394141827 2024-11-12T06:49:07,064 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ac5646bd58f44b2bd96d36627e32694, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731394144227 2024-11-12T06:49:07,075 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:07,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112289122a1125543119098c558dc03a2a5_a474a06785ead13d749016bf6309b74c is 50, key is test_row_0/A:col10/1731394145373/Put/seqid=0 2024-11-12T06:49:07,077 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112ae2ace654d20419ba25a3d47bdf696a5_a474a06785ead13d749016bf6309b74c store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:07,078 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a474a06785ead13d749016bf6309b74c#B#compaction#308 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:07,078 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/bc9f7a3b2975487594cfd41994e8d826 is 50, key is test_row_0/B:col10/1731394145363/Put/seqid=0 2024-11-12T06:49:07,079 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112ae2ace654d20419ba25a3d47bdf696a5_a474a06785ead13d749016bf6309b74c, store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:07,079 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ae2ace654d20419ba25a3d47bdf696a5_a474a06785ead13d749016bf6309b74c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T06:49:07,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742194_1370 (size=12643) 2024-11-12T06:49:07,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742193_1369 (size=12454) 2024-11-12T06:49:07,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742195_1371 (size=4469) 2024-11-12T06:49:07,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:07,486 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a474a06785ead13d749016bf6309b74c#A#compaction#307 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:07,487 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/d5c8cd2d96fb4139ba70e5a2ea69865b is 175, key is test_row_0/A:col10/1731394145363/Put/seqid=0 2024-11-12T06:49:07,488 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112289122a1125543119098c558dc03a2a5_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112289122a1125543119098c558dc03a2a5_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:07,488 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/bc9f7a3b2975487594cfd41994e8d826 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/bc9f7a3b2975487594cfd41994e8d826 2024-11-12T06:49:07,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/19d42c2b52a548efa6bed6eade339f30, store: [table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:07,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/19d42c2b52a548efa6bed6eade339f30 is 175, key is test_row_0/A:col10/1731394145373/Put/seqid=0 2024-11-12T06:49:07,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742196_1372 (size=31597) 2024-11-12T06:49:07,497 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a474a06785ead13d749016bf6309b74c/B of a474a06785ead13d749016bf6309b74c into bc9f7a3b2975487594cfd41994e8d826(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:07,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742197_1373 (size=31255) 2024-11-12T06:49:07,497 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:07,497 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., storeName=a474a06785ead13d749016bf6309b74c/B, priority=13, startTime=1731394147058; duration=0sec 2024-11-12T06:49:07,497 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:07,497 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a474a06785ead13d749016bf6309b74c:B 2024-11-12T06:49:07,497 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:07,498 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:07,498 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): a474a06785ead13d749016bf6309b74c/C is initiating minor compaction (all files) 2024-11-12T06:49:07,498 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a474a06785ead13d749016bf6309b74c/C in TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:07,498 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/6fdabf840350465280294d393a67f0bb, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/6e3709f4cb92459bacfe2fc42b303f1b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/9ecbef4dfa2e4ebe80982c26b92eb7e6] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp, totalSize=36.0 K 2024-11-12T06:49:07,498 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fdabf840350465280294d393a67f0bb, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731394141462 2024-11-12T06:49:07,499 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e3709f4cb92459bacfe2fc42b303f1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731394141827 2024-11-12T06:49:07,499 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ecbef4dfa2e4ebe80982c26b92eb7e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731394144227 2024-11-12T06:49:07,502 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/d5c8cd2d96fb4139ba70e5a2ea69865b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/d5c8cd2d96fb4139ba70e5a2ea69865b 2024-11-12T06:49:07,508 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a474a06785ead13d749016bf6309b74c#C#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:07,509 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/2c29106e6b904125bb59502a019803f5 is 50, key is test_row_0/C:col10/1731394145363/Put/seqid=0 2024-11-12T06:49:07,511 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a474a06785ead13d749016bf6309b74c/A of a474a06785ead13d749016bf6309b74c into d5c8cd2d96fb4139ba70e5a2ea69865b(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:07,511 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:07,511 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., storeName=a474a06785ead13d749016bf6309b74c/A, priority=13, startTime=1731394147057; duration=0sec 2024-11-12T06:49:07,511 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:07,511 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a474a06785ead13d749016bf6309b74c:A 2024-11-12T06:49:07,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742198_1374 (size=12643) 2024-11-12T06:49:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:07,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:07,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T06:49:07,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394147589, max timestamp=1731394147635Num uniques 859; 2024-11-12T06:49:07,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:07,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394147589, max timestamp=1731394147635Num uniques 859; 2024-11-12T06:49:07,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:07,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394147589, max timestamp=1731394147635Num uniques 859; 2024-11-12T06:49:07,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:07,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731394147696, max timestamp=1731394147715Num uniques 1068; 2024-11-12T06:49:07,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T06:49:07,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731394147696, max timestamp=1731394147715Num uniques 1068; 2024-11-12T06:49:07,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T06:49:07,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731394147696, max timestamp=1731394147715Num uniques 1068; 2024-11-12T06:49:07,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T06:49:07,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731394147771, max timestamp=1731394147792Num uniques 1077; 2024-11-12T06:49:07,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731394147771, max timestamp=1731394147792Num uniques 1077; 2024-11-12T06:49:07,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731394147771, max timestamp=1731394147792Num uniques 1077; 2024-11-12T06:49:07,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-12T06:49:07,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-12T06:49:07,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-12T06:49:07,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731394147839, max timestamp=1731394147872Num uniques 1086; 2024-11-12T06:49:07,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-12T06:49:07,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731394147839, max timestamp=1731394147872Num uniques 1086; 2024-11-12T06:49:07,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-12T06:49:07,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731394147854, max timestamp=1731394147872Num uniques 1086; 2024-11-12T06:49:07,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-12T06:49:07,897 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/19d42c2b52a548efa6bed6eade339f30 2024-11-12T06:49:07,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/25b18d0accf04138a5ac9015a6011a7c is 50, key is test_row_0/B:col10/1731394145373/Put/seqid=0 2024-11-12T06:49:07,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742199_1375 (size=12301) 2024-11-12T06:49:07,928 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/2c29106e6b904125bb59502a019803f5 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/2c29106e6b904125bb59502a019803f5 2024-11-12T06:49:07,932 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a474a06785ead13d749016bf6309b74c/C of a474a06785ead13d749016bf6309b74c into 2c29106e6b904125bb59502a019803f5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:07,932 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:07,932 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., storeName=a474a06785ead13d749016bf6309b74c/C, priority=13, startTime=1731394147058; duration=0sec 2024-11-12T06:49:07,932 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:07,932 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a474a06785ead13d749016bf6309b74c:C 2024-11-12T06:49:07,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731394147936, max timestamp=1731394147955Num uniques 1045; 2024-11-12T06:49:07,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-12T06:49:07,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731394147936, max timestamp=1731394147955Num uniques 1095; 2024-11-12T06:49:07,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-12T06:49:07,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731394147936, max timestamp=1731394147955Num uniques 1095; 2024-11-12T06:49:07,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-12T06:49:08,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731394148037, max timestamp=1731394148060Num uniques 1154; 2024-11-12T06:49:08,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-11-12T06:49:08,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731394148037, max timestamp=1731394148060Num uniques 1154; 2024-11-12T06:49:08,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-11-12T06:49:08,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731394148037, max timestamp=1731394148060Num uniques 1154; 2024-11-12T06:49:08,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-11-12T06:49:08,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1731394148086, max timestamp=1731394148134Num uniques 1013; 2024-11-12T06:49:08,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=12038, suffix heap size=133320,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=263 2024-11-12T06:49:08,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1731394148086, max timestamp=1731394148134Num uniques 1013; 2024-11-12T06:49:08,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=12038, suffix heap size=133320,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=263 2024-11-12T06:49:08,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1731394148086, max timestamp=1731394148134Num uniques 1013; 2024-11-12T06:49:08,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=12038, suffix heap size=133320,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=263 2024-11-12T06:49:08,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731394148178, max timestamp=1731394148194Num uniques 1122; 2024-11-12T06:49:08,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731394148178, max timestamp=1731394148194Num uniques 1122; 2024-11-12T06:49:08,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=10162, suffix heap size=144656,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=222 2024-11-12T06:49:08,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731394148178, max timestamp=1731394148194Num uniques 1122; 2024-11-12T06:49:08,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=10162, suffix heap size=144656,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=222 2024-11-12T06:49:08,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=10162, suffix heap size=144656,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=222 2024-11-12T06:49:08,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731394148236, max timestamp=1731394148266Num uniques 1081; 2024-11-12T06:49:08,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-12T06:49:08,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731394148236, max timestamp=1731394148266Num uniques 1081; 2024-11-12T06:49:08,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-12T06:49:08,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731394148236, max timestamp=1731394148266Num uniques 1081; 2024-11-12T06:49:08,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-12T06:49:08,306 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/25b18d0accf04138a5ac9015a6011a7c 2024-11-12T06:49:08,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/176a25f7ec134d81a2704498286d692b is 50, key is test_row_0/C:col10/1731394145373/Put/seqid=0 2024-11-12T06:49:08,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742200_1376 (size=12301) 2024-11-12T06:49:08,327 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/176a25f7ec134d81a2704498286d692b 2024-11-12T06:49:08,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/19d42c2b52a548efa6bed6eade339f30 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/19d42c2b52a548efa6bed6eade339f30 2024-11-12T06:49:08,335 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/19d42c2b52a548efa6bed6eade339f30, entries=150, sequenceid=285, filesize=30.5 K 2024-11-12T06:49:08,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/25b18d0accf04138a5ac9015a6011a7c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/25b18d0accf04138a5ac9015a6011a7c 2024-11-12T06:49:08,342 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/25b18d0accf04138a5ac9015a6011a7c, entries=150, sequenceid=285, filesize=12.0 K 2024-11-12T06:49:08,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/176a25f7ec134d81a2704498286d692b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/176a25f7ec134d81a2704498286d692b 2024-11-12T06:49:08,346 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/176a25f7ec134d81a2704498286d692b, entries=150, sequenceid=285, filesize=12.0 K 2024-11-12T06:49:08,347 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=140.89 KB/144270 for a474a06785ead13d749016bf6309b74c in 1287ms, sequenceid=285, compaction requested=false 2024-11-12T06:49:08,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:08,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:08,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-12T06:49:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-12T06:49:08,350 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-12T06:49:08,350 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8980 sec 2024-11-12T06:49:08,351 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.9020 sec 2024-11-12T06:49:08,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a474a06785ead13d749016bf6309b74c 3/3 column families, dataSize=147.60 KB heapSize=418.78 KB 2024-11-12T06:49:08,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=A 2024-11-12T06:49:08,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:49:08,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=B 2024-11-12T06:49:08,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:49:08,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=C 2024-11-12T06:49:08,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:49:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:08,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126de05bb87e3d49b39bf805e580772b14_a474a06785ead13d749016bf6309b74c is 50, key is test_row_0/A:col10/1731394148344/Put/seqid=0 2024-11-12T06:49:08,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742201_1377 (size=14994) 2024-11-12T06:49:08,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394208376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:08,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:08,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:08,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:08,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394208380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:08,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:08,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:08,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:08,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394208447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:08,455 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_1' in a474a06785ead13d749016bf6309b74c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:49:08,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:08,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:08,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:08,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:08,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394208457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:08,468 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_1' in a474a06785ead13d749016bf6309b74c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:49:08,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:08,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:08,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:08,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:08,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394208467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:08,476 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_0' in a474a06785ead13d749016bf6309b74c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:49:08,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:08,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:08,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:08,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394208487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:08,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:08,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394208497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:08,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:08,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:08,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T06:49:08,598 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:49:08,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:49:08,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-12T06:49:08,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T06:49:08,600 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:49:08,600 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:49:08,600 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:49:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T06:49:08,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:08,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394208697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:08,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:08,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394208716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:08,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:08,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:08,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:08,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T06:49:08,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:08,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:08,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:08,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:08,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:08,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:08,767 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:08,770 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126de05bb87e3d49b39bf805e580772b14_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126de05bb87e3d49b39bf805e580772b14_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:08,770 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/34a1ebe265f046eba96f1e4ce0ecd78b, store: [table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:08,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/34a1ebe265f046eba96f1e4ce0ecd78b is 175, key is test_row_0/A:col10/1731394148344/Put/seqid=0 2024-11-12T06:49:08,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742202_1378 (size=39949) 2024-11-12T06:49:08,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T06:49:08,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:08,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:08,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:08,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:08,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T06:49:09,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394209017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:09,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:09,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:09,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:09,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394209027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:09,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:09,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:09,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:09,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T06:49:09,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:09,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:09,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:09,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:09,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:09,175 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=463, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/34a1ebe265f046eba96f1e4ce0ecd78b 2024-11-12T06:49:09,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/97355ac052fc459fa1826f88060fd8e9 is 50, key is test_row_0/B:col10/1731394148344/Put/seqid=0 2024-11-12T06:49:09,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742203_1379 (size=12301) 2024-11-12T06:49:09,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T06:49:09,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:09,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:09,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:09,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:09,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T06:49:09,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T06:49:09,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:09,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:09,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:09,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:09,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:09,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T06:49:09,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:09,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:09,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:09,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:09,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:09,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394209527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:09,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:09,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394209537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:09,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:09,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:09,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:09,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/97355ac052fc459fa1826f88060fd8e9 2024-11-12T06:49:09,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/c61c7bee36364a05ac443bece86464d8 is 50, key is test_row_0/C:col10/1731394148344/Put/seqid=0 2024-11-12T06:49:09,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742204_1380 (size=12301) 2024-11-12T06:49:09,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T06:49:09,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:09,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:09,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:09,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:09,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T06:49:09,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T06:49:09,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:09,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:09,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:09,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:09,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:09,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T06:49:09,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:09,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:09,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:09,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:09,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:09,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:09,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/c61c7bee36364a05ac443bece86464d8 2024-11-12T06:49:09,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/34a1ebe265f046eba96f1e4ce0ecd78b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/34a1ebe265f046eba96f1e4ce0ecd78b 2024-11-12T06:49:10,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/34a1ebe265f046eba96f1e4ce0ecd78b, entries=200, sequenceid=463, filesize=39.0 K 2024-11-12T06:49:10,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/97355ac052fc459fa1826f88060fd8e9 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/97355ac052fc459fa1826f88060fd8e9 2024-11-12T06:49:10,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/97355ac052fc459fa1826f88060fd8e9, entries=150, sequenceid=463, filesize=12.0 K 2024-11-12T06:49:10,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/c61c7bee36364a05ac443bece86464d8 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/c61c7bee36364a05ac443bece86464d8 2024-11-12T06:49:10,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/c61c7bee36364a05ac443bece86464d8, entries=150, sequenceid=463, filesize=12.0 K 2024-11-12T06:49:10,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~418.73 KB/428784, currentSize=33.54 KB/34350 for a474a06785ead13d749016bf6309b74c in 1656ms, sequenceid=463, compaction requested=true 2024-11-12T06:49:10,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:10,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a474a06785ead13d749016bf6309b74c:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:49:10,011 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:10,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:10,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a474a06785ead13d749016bf6309b74c:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:49:10,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:10,011 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:10,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a474a06785ead13d749016bf6309b74c:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:49:10,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:10,013 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102801 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:10,013 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:10,013 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): a474a06785ead13d749016bf6309b74c/A is initiating minor compaction (all files) 2024-11-12T06:49:10,013 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): a474a06785ead13d749016bf6309b74c/B is initiating minor compaction (all files) 2024-11-12T06:49:10,013 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a474a06785ead13d749016bf6309b74c/A in TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:10,013 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a474a06785ead13d749016bf6309b74c/B in TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:10,013 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/d5c8cd2d96fb4139ba70e5a2ea69865b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/19d42c2b52a548efa6bed6eade339f30, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/34a1ebe265f046eba96f1e4ce0ecd78b] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp, totalSize=100.4 K 2024-11-12T06:49:10,013 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/bc9f7a3b2975487594cfd41994e8d826, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/25b18d0accf04138a5ac9015a6011a7c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/97355ac052fc459fa1826f88060fd8e9] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp, totalSize=36.4 K 2024-11-12T06:49:10,013 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:10,013 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. files: [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/d5c8cd2d96fb4139ba70e5a2ea69865b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/19d42c2b52a548efa6bed6eade339f30, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/34a1ebe265f046eba96f1e4ce0ecd78b] 2024-11-12T06:49:10,014 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting bc9f7a3b2975487594cfd41994e8d826, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731394144227 2024-11-12T06:49:10,014 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5c8cd2d96fb4139ba70e5a2ea69865b, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731394144227 2024-11-12T06:49:10,014 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 25b18d0accf04138a5ac9015a6011a7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731394145365 2024-11-12T06:49:10,014 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19d42c2b52a548efa6bed6eade339f30, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731394145365 2024-11-12T06:49:10,014 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 97355ac052fc459fa1826f88060fd8e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731394148340 2024-11-12T06:49:10,014 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34a1ebe265f046eba96f1e4ce0ecd78b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731394148298 2024-11-12T06:49:10,020 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:10,021 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a474a06785ead13d749016bf6309b74c#B#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:10,021 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112ed2cf0f5d9a14227b3bdc5a3b621eba0_a474a06785ead13d749016bf6309b74c store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:10,022 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/7ab418f86a244d80a460564d96aa309b is 50, key is test_row_0/B:col10/1731394148344/Put/seqid=0 2024-11-12T06:49:10,023 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112ed2cf0f5d9a14227b3bdc5a3b621eba0_a474a06785ead13d749016bf6309b74c, store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:10,023 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ed2cf0f5d9a14227b3bdc5a3b621eba0_a474a06785ead13d749016bf6309b74c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:10,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742205_1381 (size=12745) 2024-11-12T06:49:10,028 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/7ab418f86a244d80a460564d96aa309b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/7ab418f86a244d80a460564d96aa309b 2024-11-12T06:49:10,032 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a474a06785ead13d749016bf6309b74c/B of a474a06785ead13d749016bf6309b74c into 7ab418f86a244d80a460564d96aa309b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:10,032 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:10,032 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., storeName=a474a06785ead13d749016bf6309b74c/B, priority=13, startTime=1731394150011; duration=0sec 2024-11-12T06:49:10,032 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:10,032 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a474a06785ead13d749016bf6309b74c:B 2024-11-12T06:49:10,032 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:10,033 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:10,033 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): a474a06785ead13d749016bf6309b74c/C is initiating minor compaction (all files) 2024-11-12T06:49:10,033 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a474a06785ead13d749016bf6309b74c/C in TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:10,033 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/2c29106e6b904125bb59502a019803f5, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/176a25f7ec134d81a2704498286d692b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/c61c7bee36364a05ac443bece86464d8] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp, totalSize=36.4 K 2024-11-12T06:49:10,033 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c29106e6b904125bb59502a019803f5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731394144227 2024-11-12T06:49:10,034 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 176a25f7ec134d81a2704498286d692b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731394145365 2024-11-12T06:49:10,034 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting c61c7bee36364a05ac443bece86464d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731394148340 2024-11-12T06:49:10,040 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a474a06785ead13d749016bf6309b74c#C#compaction#317 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:10,040 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/d017c25c7f204ae9bf997dc348e39a0e is 50, key is test_row_0/C:col10/1731394148344/Put/seqid=0 2024-11-12T06:49:10,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742206_1382 (size=4469) 2024-11-12T06:49:10,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742207_1383 (size=12745) 2024-11-12T06:49:10,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T06:49:10,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:10,126 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing a474a06785ead13d749016bf6309b74c 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-12T06:49:10,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=A 2024-11-12T06:49:10,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:10,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=B 2024-11-12T06:49:10,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:10,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=C 2024-11-12T06:49:10,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:10,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112450caf6ff0cc458d850299ab5eb6f19a_a474a06785ead13d749016bf6309b74c is 50, key is test_row_0/A:col10/1731394148367/Put/seqid=0 2024-11-12T06:49:10,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742208_1384 (size=12454) 2024-11-12T06:49:10,445 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a474a06785ead13d749016bf6309b74c#A#compaction#316 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:10,445 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/851d8d37525748279b182e2b7d21fdf6 is 175, key is test_row_0/A:col10/1731394148344/Put/seqid=0 2024-11-12T06:49:10,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742209_1385 (size=31699) 2024-11-12T06:49:10,458 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/d017c25c7f204ae9bf997dc348e39a0e as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/d017c25c7f204ae9bf997dc348e39a0e 2024-11-12T06:49:10,459 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/851d8d37525748279b182e2b7d21fdf6 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/851d8d37525748279b182e2b7d21fdf6 2024-11-12T06:49:10,463 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a474a06785ead13d749016bf6309b74c/C of a474a06785ead13d749016bf6309b74c into d017c25c7f204ae9bf997dc348e39a0e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:10,463 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:10,463 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., storeName=a474a06785ead13d749016bf6309b74c/C, priority=13, startTime=1731394150011; duration=0sec 2024-11-12T06:49:10,463 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:10,463 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a474a06785ead13d749016bf6309b74c:C 2024-11-12T06:49:10,466 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a474a06785ead13d749016bf6309b74c/A of a474a06785ead13d749016bf6309b74c into 851d8d37525748279b182e2b7d21fdf6(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:10,466 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:10,467 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., storeName=a474a06785ead13d749016bf6309b74c/A, priority=13, startTime=1731394150011; duration=0sec 2024-11-12T06:49:10,467 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:10,467 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a474a06785ead13d749016bf6309b74c:A 2024-11-12T06:49:10,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:10,543 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112450caf6ff0cc458d850299ab5eb6f19a_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112450caf6ff0cc458d850299ab5eb6f19a_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:10,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/c592171a80e342d3867ad773b529214c, store: [table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:10,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/c592171a80e342d3867ad773b529214c is 175, key is test_row_0/A:col10/1731394148367/Put/seqid=0 2024-11-12T06:49:10,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742210_1386 (size=31255) 2024-11-12T06:49:10,551 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=472, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/c592171a80e342d3867ad773b529214c 2024-11-12T06:49:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:10,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:10,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/94ccfaffb17843fb8b32d20ace6e6efb is 50, key is test_row_0/B:col10/1731394148367/Put/seqid=0 2024-11-12T06:49:10,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742211_1387 (size=12301) 2024-11-12T06:49:10,575 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/94ccfaffb17843fb8b32d20ace6e6efb 2024-11-12T06:49:10,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/fdcd7400a5ab47e89e8a689f5d1d5ca4 is 50, key is test_row_0/C:col10/1731394148367/Put/seqid=0 2024-11-12T06:49:10,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742212_1388 (size=12301) 2024-11-12T06:49:10,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394150589, max timestamp=1731394150633Num uniques 859; 2024-11-12T06:49:10,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394150589, max timestamp=1731394150633Num uniques 859; 2024-11-12T06:49:10,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:10,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:10,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394150589, max timestamp=1731394150633Num uniques 859; 2024-11-12T06:49:10,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:10,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394210719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:10,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:10,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:10,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:10,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731394150699, max timestamp=1731394150731Num uniques 1068; 2024-11-12T06:49:10,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T06:49:10,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731394150699, max timestamp=1731394150731Num uniques 1068; 2024-11-12T06:49:10,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T06:49:10,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731394150699, max timestamp=1731394150731Num uniques 1068; 2024-11-12T06:49:10,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T06:49:10,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T06:49:10,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394210854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:10,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394210865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:10,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:10,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:10,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:10,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731394150776, max timestamp=1731394150863Num uniques 1077; 2024-11-12T06:49:10,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-12T06:49:10,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731394150776, max timestamp=1731394150863Num uniques 1077; 2024-11-12T06:49:10,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-12T06:49:10,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731394150776, max timestamp=1731394150863Num uniques 1077; 2024-11-12T06:49:10,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-12T06:49:10,988 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/fdcd7400a5ab47e89e8a689f5d1d5ca4 2024-11-12T06:49:10,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/c592171a80e342d3867ad773b529214c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/c592171a80e342d3867ad773b529214c 2024-11-12T06:49:10,996 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/c592171a80e342d3867ad773b529214c, entries=150, sequenceid=472, filesize=30.5 K 2024-11-12T06:49:10,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/94ccfaffb17843fb8b32d20ace6e6efb as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/94ccfaffb17843fb8b32d20ace6e6efb 2024-11-12T06:49:10,999 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/94ccfaffb17843fb8b32d20ace6e6efb, entries=150, sequenceid=472, filesize=12.0 K 2024-11-12T06:49:11,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/fdcd7400a5ab47e89e8a689f5d1d5ca4 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/fdcd7400a5ab47e89e8a689f5d1d5ca4 2024-11-12T06:49:11,003 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/fdcd7400a5ab47e89e8a689f5d1d5ca4, entries=150, sequenceid=472, filesize=12.0 K 2024-11-12T06:49:11,004 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=80.51 KB/82440 for a474a06785ead13d749016bf6309b74c in 878ms, sequenceid=472, compaction requested=false 2024-11-12T06:49:11,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:11,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:11,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-12T06:49:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-12T06:49:11,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-12T06:49:11,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4050 sec 2024-11-12T06:49:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:11,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a474a06785ead13d749016bf6309b74c 3/3 column families, dataSize=93.93 KB heapSize=254.39 KB 2024-11-12T06:49:11,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=A 2024-11-12T06:49:11,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:11,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=B 2024-11-12T06:49:11,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:11,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=C 2024-11-12T06:49:11,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:11,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.4090 sec 2024-11-12T06:49:11,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128eb5a61ee8d44c98ae88d6ccb31d80c2_a474a06785ead13d749016bf6309b74c is 50, key is test_row_0/A:col10/1731394151006/Put/seqid=0 2024-11-12T06:49:11,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742213_1389 (size=17534) 2024-11-12T06:49:11,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394211066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:11,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:11,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:11,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:11,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394211067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:11,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394211177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:11,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:11,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:11,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:11,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:11,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394211187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:11,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:11,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:11,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:11,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394211387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:11,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:11,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:11,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:11,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394211407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:11,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:11,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:11,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:11,417 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:11,420 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128eb5a61ee8d44c98ae88d6ccb31d80c2_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128eb5a61ee8d44c98ae88d6ccb31d80c2_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:11,421 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/680629f027ab42fb8a0929a302724aeb, store: [table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:11,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/680629f027ab42fb8a0929a302724aeb is 175, key is test_row_0/A:col10/1731394151006/Put/seqid=0 2024-11-12T06:49:11,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742214_1390 (size=48639) 2024-11-12T06:49:11,424 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=537, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/680629f027ab42fb8a0929a302724aeb 2024-11-12T06:49:11,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/dfc60af1470c474cb363dd88940c8ac2 is 50, key is test_row_0/B:col10/1731394151006/Put/seqid=0 2024-11-12T06:49:11,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742215_1391 (size=12301) 2024-11-12T06:49:11,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:11,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394211697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:11,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:11,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:11,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:11,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394211718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:11,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/dfc60af1470c474cb363dd88940c8ac2 2024-11-12T06:49:11,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/d3367f0920a040138568ae6bb67b9a6c is 50, key is test_row_0/C:col10/1731394151006/Put/seqid=0 2024-11-12T06:49:11,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742216_1392 (size=12301) 2024-11-12T06:49:11,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/d3367f0920a040138568ae6bb67b9a6c 2024-11-12T06:49:11,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/680629f027ab42fb8a0929a302724aeb as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/680629f027ab42fb8a0929a302724aeb 2024-11-12T06:49:11,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/680629f027ab42fb8a0929a302724aeb, entries=250, sequenceid=537, filesize=47.5 K 2024-11-12T06:49:11,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/dfc60af1470c474cb363dd88940c8ac2 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/dfc60af1470c474cb363dd88940c8ac2 2024-11-12T06:49:11,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/dfc60af1470c474cb363dd88940c8ac2, entries=150, sequenceid=537, filesize=12.0 K 2024-11-12T06:49:11,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/d3367f0920a040138568ae6bb67b9a6c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/d3367f0920a040138568ae6bb67b9a6c 2024-11-12T06:49:11,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/d3367f0920a040138568ae6bb67b9a6c, entries=150, sequenceid=537, filesize=12.0 K 2024-11-12T06:49:11,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~254.34 KB/260448, currentSize=87.22 KB/89310 for a474a06785ead13d749016bf6309b74c in 862ms, sequenceid=537, compaction requested=true 2024-11-12T06:49:11,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:11,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a474a06785ead13d749016bf6309b74c:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:49:11,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:11,869 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:11,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a474a06785ead13d749016bf6309b74c:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:49:11,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:11,869 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:11,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a474a06785ead13d749016bf6309b74c:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:49:11,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:11,870 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:11,870 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111593 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:11,870 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): a474a06785ead13d749016bf6309b74c/A is initiating minor compaction (all files) 2024-11-12T06:49:11,870 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): a474a06785ead13d749016bf6309b74c/B is initiating minor compaction (all files) 2024-11-12T06:49:11,870 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a474a06785ead13d749016bf6309b74c/A in TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:11,870 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a474a06785ead13d749016bf6309b74c/B in TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:11,870 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/851d8d37525748279b182e2b7d21fdf6, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/c592171a80e342d3867ad773b529214c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/680629f027ab42fb8a0929a302724aeb] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp, totalSize=109.0 K 2024-11-12T06:49:11,870 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/7ab418f86a244d80a460564d96aa309b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/94ccfaffb17843fb8b32d20ace6e6efb, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/dfc60af1470c474cb363dd88940c8ac2] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp, totalSize=36.5 K 2024-11-12T06:49:11,870 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:11,870 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. files: [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/851d8d37525748279b182e2b7d21fdf6, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/c592171a80e342d3867ad773b529214c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/680629f027ab42fb8a0929a302724aeb] 2024-11-12T06:49:11,870 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ab418f86a244d80a460564d96aa309b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731394148340 2024-11-12T06:49:11,871 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 851d8d37525748279b182e2b7d21fdf6, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731394148340 2024-11-12T06:49:11,871 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 94ccfaffb17843fb8b32d20ace6e6efb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731394148367 2024-11-12T06:49:11,871 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting dfc60af1470c474cb363dd88940c8ac2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731394150987 2024-11-12T06:49:11,871 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting c592171a80e342d3867ad773b529214c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731394148367 2024-11-12T06:49:11,871 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 680629f027ab42fb8a0929a302724aeb, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731394150987 2024-11-12T06:49:11,877 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:11,878 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a474a06785ead13d749016bf6309b74c#B#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:11,878 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/95e3c62f802f4960874e2b37f125b9b8 is 50, key is test_row_0/B:col10/1731394151006/Put/seqid=0 2024-11-12T06:49:11,879 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112b33bbe65e3ae4c6388eb649dcb8458ce_a474a06785ead13d749016bf6309b74c store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:11,882 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112b33bbe65e3ae4c6388eb649dcb8458ce_a474a06785ead13d749016bf6309b74c, store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:11,882 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b33bbe65e3ae4c6388eb649dcb8458ce_a474a06785ead13d749016bf6309b74c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:11,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742217_1393 (size=12847) 2024-11-12T06:49:11,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742218_1394 (size=4469) 2024-11-12T06:49:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:12,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a474a06785ead13d749016bf6309b74c 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T06:49:12,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=A 2024-11-12T06:49:12,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:12,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=B 2024-11-12T06:49:12,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:12,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=C 2024-11-12T06:49:12,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:12,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128f2e6cc8001846b8874e2d354ff536d4_a474a06785ead13d749016bf6309b74c is 50, key is test_row_0/A:col10/1731394151039/Put/seqid=0 2024-11-12T06:49:12,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742219_1395 (size=14994) 2024-11-12T06:49:12,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394212266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:12,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:12,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:12,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:12,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394212274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:12,291 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/95e3c62f802f4960874e2b37f125b9b8 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/95e3c62f802f4960874e2b37f125b9b8 2024-11-12T06:49:12,295 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a474a06785ead13d749016bf6309b74c/B of a474a06785ead13d749016bf6309b74c into 95e3c62f802f4960874e2b37f125b9b8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:12,295 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:12,295 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., storeName=a474a06785ead13d749016bf6309b74c/B, priority=13, startTime=1731394151869; duration=0sec 2024-11-12T06:49:12,295 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:12,295 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a474a06785ead13d749016bf6309b74c:B 2024-11-12T06:49:12,295 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:12,295 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a474a06785ead13d749016bf6309b74c#A#compaction#325 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:12,296 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:12,296 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): a474a06785ead13d749016bf6309b74c/C is initiating minor compaction (all files) 2024-11-12T06:49:12,296 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a474a06785ead13d749016bf6309b74c/C in TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:12,296 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/9232c617a16d45579fa2ea90a04fa27c is 175, key is test_row_0/A:col10/1731394151006/Put/seqid=0 2024-11-12T06:49:12,296 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/d017c25c7f204ae9bf997dc348e39a0e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/fdcd7400a5ab47e89e8a689f5d1d5ca4, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/d3367f0920a040138568ae6bb67b9a6c] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp, totalSize=36.5 K 2024-11-12T06:49:12,296 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting d017c25c7f204ae9bf997dc348e39a0e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731394148340 2024-11-12T06:49:12,296 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting fdcd7400a5ab47e89e8a689f5d1d5ca4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731394148367 2024-11-12T06:49:12,297 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting d3367f0920a040138568ae6bb67b9a6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731394150987 2024-11-12T06:49:12,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742220_1396 (size=31801) 2024-11-12T06:49:12,302 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a474a06785ead13d749016bf6309b74c#C#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:12,302 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/dd0b1c5a52fb4da3a4e4222cd281a578 is 50, key is test_row_0/C:col10/1731394151006/Put/seqid=0 2024-11-12T06:49:12,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742221_1397 (size=12847) 2024-11-12T06:49:12,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394212377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:12,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:12,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:12,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:12,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394212387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:12,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1731394212497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:12,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731394212497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:12,505 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_1' in a474a06785ead13d749016bf6309b74c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:49:12,505 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_1' in a474a06785ead13d749016bf6309b74c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:49:12,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:12,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:12,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:12,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:12,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:12,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:12,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49496 deadline: 1731394212517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:12,525 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_0' in a474a06785ead13d749016bf6309b74c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8272 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:49:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:12,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394212597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:12,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:12,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394212597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:12,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:12,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:12,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:12,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:12,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:12,633 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:12,636 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128f2e6cc8001846b8874e2d354ff536d4_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128f2e6cc8001846b8874e2d354ff536d4_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:12,637 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/1a86abfd10154a9e834277706bdefb1f, store: [table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:12,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/1a86abfd10154a9e834277706bdefb1f is 175, key is test_row_0/A:col10/1731394151039/Put/seqid=0 2024-11-12T06:49:12,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742222_1398 (size=39949) 2024-11-12T06:49:12,703 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/9232c617a16d45579fa2ea90a04fa27c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/9232c617a16d45579fa2ea90a04fa27c 2024-11-12T06:49:12,707 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a474a06785ead13d749016bf6309b74c/A of a474a06785ead13d749016bf6309b74c into 9232c617a16d45579fa2ea90a04fa27c(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:12,707 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:12,707 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., storeName=a474a06785ead13d749016bf6309b74c/A, priority=13, startTime=1731394151869; duration=0sec 2024-11-12T06:49:12,707 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:12,707 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a474a06785ead13d749016bf6309b74c:A 2024-11-12T06:49:12,716 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/dd0b1c5a52fb4da3a4e4222cd281a578 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/dd0b1c5a52fb4da3a4e4222cd281a578 2024-11-12T06:49:12,720 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a474a06785ead13d749016bf6309b74c/C of a474a06785ead13d749016bf6309b74c into dd0b1c5a52fb4da3a4e4222cd281a578(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:12,720 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:12,720 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., storeName=a474a06785ead13d749016bf6309b74c/C, priority=13, startTime=1731394151869; duration=0sec 2024-11-12T06:49:12,720 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:12,720 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a474a06785ead13d749016bf6309b74c:C 2024-11-12T06:49:12,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T06:49:12,767 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:49:12,768 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:49:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-12T06:49:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T06:49:12,770 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:49:12,770 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:49:12,770 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:49:12,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T06:49:12,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T06:49:12,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:12,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:12,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:12,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:12,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:12,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:12,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:12,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394212917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:12,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:12,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:12,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:12,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394212917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:12,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:12,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:12,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:12,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:13,040 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=554, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/1a86abfd10154a9e834277706bdefb1f 2024-11-12T06:49:13,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/a99f13054f7a4836878c1a7b2e155f92 is 50, key is test_row_0/B:col10/1731394151039/Put/seqid=0 2024-11-12T06:49:13,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742223_1399 (size=12301) 2024-11-12T06:49:13,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T06:49:13,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:13,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:13,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:13,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:13,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:13,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:13,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T06:49:13,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T06:49:13,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:13,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:13,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:13,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:13,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:13,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:13,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T06:49:13,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:13,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:13,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:13,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:13,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T06:49:13,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:13,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394213437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:13,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:13,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:13,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394213437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:13,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:13,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:13,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:13,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:13,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:13,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/a99f13054f7a4836878c1a7b2e155f92 2024-11-12T06:49:13,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/443f7c40e13f4ad285c95b85ce2826f5 is 50, key is test_row_0/C:col10/1731394151039/Put/seqid=0 2024-11-12T06:49:13,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742224_1400 (size=12301) 2024-11-12T06:49:13,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T06:49:13,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:13,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:13,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:13,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:13,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:13,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:13,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T06:49:13,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:13,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:13,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:13,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:13,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:13,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:13,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T06:49:13,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:13,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:13,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:13,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:13,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:13,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:13,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/443f7c40e13f4ad285c95b85ce2826f5 2024-11-12T06:49:13,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/1a86abfd10154a9e834277706bdefb1f as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/1a86abfd10154a9e834277706bdefb1f 2024-11-12T06:49:13,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/1a86abfd10154a9e834277706bdefb1f, entries=200, sequenceid=554, filesize=39.0 K 2024-11-12T06:49:13,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/a99f13054f7a4836878c1a7b2e155f92 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/a99f13054f7a4836878c1a7b2e155f92 2024-11-12T06:49:13,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/a99f13054f7a4836878c1a7b2e155f92, entries=150, sequenceid=554, filesize=12.0 K 2024-11-12T06:49:13,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/443f7c40e13f4ad285c95b85ce2826f5 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/443f7c40e13f4ad285c95b85ce2826f5 2024-11-12T06:49:13,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/443f7c40e13f4ad285c95b85ce2826f5, entries=150, sequenceid=554, filesize=12.0 K 2024-11-12T06:49:13,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for a474a06785ead13d749016bf6309b74c in 1659ms, sequenceid=554, compaction requested=false 2024-11-12T06:49:13,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T06:49:13,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T06:49:13,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:13,991 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing a474a06785ead13d749016bf6309b74c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T06:49:13,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=A 2024-11-12T06:49:13,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:13,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=B 2024-11-12T06:49:13,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:13,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=C 2024-11-12T06:49:13,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:13,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128d323f2dccea438f87a2c68292219739_a474a06785ead13d749016bf6309b74c is 50, key is test_row_0/A:col10/1731394152265/Put/seqid=0 2024-11-12T06:49:14,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742225_1401 (size=12454) 2024-11-12T06:49:14,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:14,405 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128d323f2dccea438f87a2c68292219739_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128d323f2dccea438f87a2c68292219739_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:14,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/53ca41832fef4629b1fb18d3080aa576, store: [table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:14,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/53ca41832fef4629b1fb18d3080aa576 is 175, key is test_row_0/A:col10/1731394152265/Put/seqid=0 2024-11-12T06:49:14,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742226_1402 (size=31255) 2024-11-12T06:49:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:14,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:14,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394214501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:14,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394214504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:14,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:14,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:14,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:14,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394214617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:14,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394214617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:14,810 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=572, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/53ca41832fef4629b1fb18d3080aa576 2024-11-12T06:49:14,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/8221bcc8b9d0417489851bbca1f4f246 is 50, key is test_row_0/B:col10/1731394152265/Put/seqid=0 2024-11-12T06:49:14,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742227_1403 (size=12301) 2024-11-12T06:49:14,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:14,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394214827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:14,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394214837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T06:49:15,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394215137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:15,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:15,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394215157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:15,220 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/8221bcc8b9d0417489851bbca1f4f246 2024-11-12T06:49:15,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/ea6ffce713ce4ec4945cc44198438203 is 50, key is test_row_0/C:col10/1731394152265/Put/seqid=0 2024-11-12T06:49:15,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742228_1404 (size=12301) 2024-11-12T06:49:15,232 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/ea6ffce713ce4ec4945cc44198438203 2024-11-12T06:49:15,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/53ca41832fef4629b1fb18d3080aa576 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/53ca41832fef4629b1fb18d3080aa576 2024-11-12T06:49:15,240 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/53ca41832fef4629b1fb18d3080aa576, entries=150, sequenceid=572, filesize=30.5 K 2024-11-12T06:49:15,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/8221bcc8b9d0417489851bbca1f4f246 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/8221bcc8b9d0417489851bbca1f4f246 2024-11-12T06:49:15,244 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/8221bcc8b9d0417489851bbca1f4f246, entries=150, sequenceid=572, filesize=12.0 K 2024-11-12T06:49:15,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/ea6ffce713ce4ec4945cc44198438203 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/ea6ffce713ce4ec4945cc44198438203 2024-11-12T06:49:15,247 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/ea6ffce713ce4ec4945cc44198438203, entries=150, sequenceid=572, filesize=12.0 K 2024-11-12T06:49:15,248 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a474a06785ead13d749016bf6309b74c in 1258ms, sequenceid=572, compaction requested=true 2024-11-12T06:49:15,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:15,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:15,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-12T06:49:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-12T06:49:15,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-12T06:49:15,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4790 sec 2024-11-12T06:49:15,252 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.4830 sec 2024-11-12T06:49:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:15,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a474a06785ead13d749016bf6309b74c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T06:49:15,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=A 2024-11-12T06:49:15,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:15,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=B 2024-11-12T06:49:15,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:15,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=C 2024-11-12T06:49:15,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:15,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123efd11a368ad43da8d1e7084ba2b0184_a474a06785ead13d749016bf6309b74c is 50, key is test_row_0/A:col10/1731394155661/Put/seqid=0 2024-11-12T06:49:15,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742229_1405 (size=14994) 2024-11-12T06:49:15,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394215713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:15,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:15,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:15,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:15,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394215714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:15,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:15,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394215827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:15,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:15,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:15,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:15,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:15,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394215837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:15,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:15,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:15,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:16,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394216047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:16,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394216047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:16,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:16,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:16,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:16,075 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:16,077 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123efd11a368ad43da8d1e7084ba2b0184_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123efd11a368ad43da8d1e7084ba2b0184_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:16,078 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/cd03d61c9ad14c8c85afc468d8d9f101, store: [table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:16,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/cd03d61c9ad14c8c85afc468d8d9f101 is 175, key is test_row_0/A:col10/1731394155661/Put/seqid=0 2024-11-12T06:49:16,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742230_1406 (size=39949) 2024-11-12T06:49:16,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394216357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:16,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:16,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394216357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:16,482 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=590, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/cd03d61c9ad14c8c85afc468d8d9f101 2024-11-12T06:49:16,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/e85cb761c28c49e3ac57810533098243 is 50, key is test_row_0/B:col10/1731394155661/Put/seqid=0 2024-11-12T06:49:16,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742231_1407 (size=12301) 2024-11-12T06:49:16,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731394216877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:16,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:16,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:16,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:16,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49494 deadline: 1731394216877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:16,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:16,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:16,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:16,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:16,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/e85cb761c28c49e3ac57810533098243 2024-11-12T06:49:16,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/87b0645e119e4394a5da8434a9f4234b is 50, key is test_row_0/C:col10/1731394155661/Put/seqid=0 2024-11-12T06:49:16,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742232_1408 (size=12301) 2024-11-12T06:49:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T06:49:16,928 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:49:16,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:49:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-12T06:49:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T06:49:16,930 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:49:16,930 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:49:16,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:49:16,971 INFO [Thread-1514 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1514. 2024-11-12T06:49:16,971 DEBUG [Thread-1514 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:49:16,971 DEBUG [Thread-1514 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:16,971 DEBUG [Thread-1514 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:16,971 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:49:16,972 INFO [Thread-1508 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1508. 2024-11-12T06:49:16,972 DEBUG [Thread-1508 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:49:16,972 DEBUG [Thread-1508 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:16,972 DEBUG [Thread-1508 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:16,972 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:49:16,974 INFO [Thread-1506 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1506. 2024-11-12T06:49:16,974 DEBUG [Thread-1506 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:49:16,974 DEBUG [Thread-1506 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:16,974 DEBUG [Thread-1506 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:16,974 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:49:16,981 INFO [Thread-1512 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1512. 2024-11-12T06:49:16,981 DEBUG [Thread-1512 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:49:16,981 DEBUG [Thread-1512 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:16,981 DEBUG [Thread-1512 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:16,981 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:49:16,983 INFO [Thread-1510 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1510. 2024-11-12T06:49:16,983 DEBUG [Thread-1510 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:49:16,983 DEBUG [Thread-1510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:16,983 DEBUG [Thread-1510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:16,983 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:49:17,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T06:49:17,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T06:49:17,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:17,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:17,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:17,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:17,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:17,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T06:49:17,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:17,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:17,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:17,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:17,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:17,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T06:49:17,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/87b0645e119e4394a5da8434a9f4234b 2024-11-12T06:49:17,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/cd03d61c9ad14c8c85afc468d8d9f101 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/cd03d61c9ad14c8c85afc468d8d9f101 2024-11-12T06:49:17,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/cd03d61c9ad14c8c85afc468d8d9f101, entries=200, sequenceid=590, filesize=39.0 K 2024-11-12T06:49:17,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/e85cb761c28c49e3ac57810533098243 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/e85cb761c28c49e3ac57810533098243 2024-11-12T06:49:17,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/e85cb761c28c49e3ac57810533098243, entries=150, sequenceid=590, filesize=12.0 K 2024-11-12T06:49:17,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/87b0645e119e4394a5da8434a9f4234b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/87b0645e119e4394a5da8434a9f4234b 2024-11-12T06:49:17,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/87b0645e119e4394a5da8434a9f4234b, entries=150, sequenceid=590, filesize=12.0 K 2024-11-12T06:49:17,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for a474a06785ead13d749016bf6309b74c in 1661ms, sequenceid=590, compaction requested=true 2024-11-12T06:49:17,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:17,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a474a06785ead13d749016bf6309b74c:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:49:17,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:17,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a474a06785ead13d749016bf6309b74c:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:49:17,325 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:49:17,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:17,325 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:49:17,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a474a06785ead13d749016bf6309b74c:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:49:17,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:17,326 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:49:17,326 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:49:17,326 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): a474a06785ead13d749016bf6309b74c/B is initiating minor compaction (all files) 2024-11-12T06:49:17,326 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): a474a06785ead13d749016bf6309b74c/A is initiating minor compaction (all files) 2024-11-12T06:49:17,326 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a474a06785ead13d749016bf6309b74c/B in TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:17,326 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a474a06785ead13d749016bf6309b74c/A in TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:17,326 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/95e3c62f802f4960874e2b37f125b9b8, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/a99f13054f7a4836878c1a7b2e155f92, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/8221bcc8b9d0417489851bbca1f4f246, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/e85cb761c28c49e3ac57810533098243] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp, totalSize=48.6 K 2024-11-12T06:49:17,326 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/9232c617a16d45579fa2ea90a04fa27c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/1a86abfd10154a9e834277706bdefb1f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/53ca41832fef4629b1fb18d3080aa576, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/cd03d61c9ad14c8c85afc468d8d9f101] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp, totalSize=139.6 K 2024-11-12T06:49:17,326 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:17,326 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. files: [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/9232c617a16d45579fa2ea90a04fa27c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/1a86abfd10154a9e834277706bdefb1f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/53ca41832fef4629b1fb18d3080aa576, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/cd03d61c9ad14c8c85afc468d8d9f101] 2024-11-12T06:49:17,326 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 95e3c62f802f4960874e2b37f125b9b8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731394150987 2024-11-12T06:49:17,326 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9232c617a16d45579fa2ea90a04fa27c, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731394150987 2024-11-12T06:49:17,326 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting a99f13054f7a4836878c1a7b2e155f92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731394151039 2024-11-12T06:49:17,326 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a86abfd10154a9e834277706bdefb1f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731394151039 2024-11-12T06:49:17,326 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 8221bcc8b9d0417489851bbca1f4f246, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1731394152264 2024-11-12T06:49:17,327 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53ca41832fef4629b1fb18d3080aa576, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1731394152264 2024-11-12T06:49:17,327 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting e85cb761c28c49e3ac57810533098243, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1731394154471 2024-11-12T06:49:17,327 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd03d61c9ad14c8c85afc468d8d9f101, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1731394154471 2024-11-12T06:49:17,334 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a474a06785ead13d749016bf6309b74c#B#compaction#336 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:17,335 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/89c96c2a940e4a9cb7a79a5c1e2f4a73 is 50, key is test_row_0/B:col10/1731394155661/Put/seqid=0 2024-11-12T06:49:17,335 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:17,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742233_1409 (size=12983) 2024-11-12T06:49:17,339 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112130b0e1fc36846d6bead4b772146da3c_a474a06785ead13d749016bf6309b74c store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:17,368 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112130b0e1fc36846d6bead4b772146da3c_a474a06785ead13d749016bf6309b74c, store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:17,368 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112130b0e1fc36846d6bead4b772146da3c_a474a06785ead13d749016bf6309b74c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:17,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742234_1410 (size=4469) 2024-11-12T06:49:17,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T06:49:17,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:17,393 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing a474a06785ead13d749016bf6309b74c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T06:49:17,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=A 2024-11-12T06:49:17,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:17,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=B 2024-11-12T06:49:17,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:17,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=C 2024-11-12T06:49:17,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:17,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112531d33f469f74bfea546ed2fbda4d59c_a474a06785ead13d749016bf6309b74c is 50, key is test_row_1/A:col10/1731394155703/Put/seqid=0 2024-11-12T06:49:17,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742235_1411 (size=9914) 2024-11-12T06:49:17,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T06:49:17,743 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/89c96c2a940e4a9cb7a79a5c1e2f4a73 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/89c96c2a940e4a9cb7a79a5c1e2f4a73 2024-11-12T06:49:17,746 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a474a06785ead13d749016bf6309b74c/B of a474a06785ead13d749016bf6309b74c into 89c96c2a940e4a9cb7a79a5c1e2f4a73(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:17,746 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:17,746 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., storeName=a474a06785ead13d749016bf6309b74c/B, priority=12, startTime=1731394157325; duration=0sec 2024-11-12T06:49:17,746 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:17,746 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a474a06785ead13d749016bf6309b74c:B 2024-11-12T06:49:17,746 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:49:17,747 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:49:17,747 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): a474a06785ead13d749016bf6309b74c/C is initiating minor compaction (all files) 2024-11-12T06:49:17,747 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a474a06785ead13d749016bf6309b74c/C in TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:17,748 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/dd0b1c5a52fb4da3a4e4222cd281a578, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/443f7c40e13f4ad285c95b85ce2826f5, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/ea6ffce713ce4ec4945cc44198438203, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/87b0645e119e4394a5da8434a9f4234b] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp, totalSize=48.6 K 2024-11-12T06:49:17,748 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting dd0b1c5a52fb4da3a4e4222cd281a578, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731394150987 2024-11-12T06:49:17,748 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 443f7c40e13f4ad285c95b85ce2826f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731394151039 2024-11-12T06:49:17,748 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting ea6ffce713ce4ec4945cc44198438203, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1731394152264 2024-11-12T06:49:17,749 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 87b0645e119e4394a5da8434a9f4234b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1731394154471 2024-11-12T06:49:17,758 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a474a06785ead13d749016bf6309b74c#C#compaction#339 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:17,758 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/12f1b8f90f4c4df2a111fd2d06dd61e6 is 50, key is test_row_0/C:col10/1731394155661/Put/seqid=0 2024-11-12T06:49:17,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742236_1412 (size=12983) 2024-11-12T06:49:17,772 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a474a06785ead13d749016bf6309b74c#A#compaction#337 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:17,773 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/a11c2a6a14374cb88f21a68b21f60412 is 175, key is test_row_0/A:col10/1731394155661/Put/seqid=0 2024-11-12T06:49:17,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742237_1413 (size=31937) 2024-11-12T06:49:17,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:17,807 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112531d33f469f74bfea546ed2fbda4d59c_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112531d33f469f74bfea546ed2fbda4d59c_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:17,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/7919aa3987394db6aed73829c777323c, store: [table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:17,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/7919aa3987394db6aed73829c777323c is 175, key is test_row_1/A:col10/1731394155703/Put/seqid=0 2024-11-12T06:49:17,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742238_1414 (size=22561) 2024-11-12T06:49:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:17,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. as already flushing 2024-11-12T06:49:17,888 INFO [Thread-1495 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1495. 2024-11-12T06:49:17,888 DEBUG [Thread-1495 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:49:17,888 DEBUG [Thread-1495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:17,888 DEBUG [Thread-1495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:17,888 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:49:17,898 INFO [Thread-1499 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1499. 2024-11-12T06:49:17,898 DEBUG [Thread-1499 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:49:17,899 DEBUG [Thread-1499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:17,899 DEBUG [Thread-1499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:17,899 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:49:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T06:49:18,170 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/12f1b8f90f4c4df2a111fd2d06dd61e6 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/12f1b8f90f4c4df2a111fd2d06dd61e6 2024-11-12T06:49:18,176 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a474a06785ead13d749016bf6309b74c/C of a474a06785ead13d749016bf6309b74c into 12f1b8f90f4c4df2a111fd2d06dd61e6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:18,176 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:18,176 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., storeName=a474a06785ead13d749016bf6309b74c/C, priority=12, startTime=1731394157325; duration=0sec 2024-11-12T06:49:18,176 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:18,176 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a474a06785ead13d749016bf6309b74c:C 2024-11-12T06:49:18,181 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/a11c2a6a14374cb88f21a68b21f60412 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/a11c2a6a14374cb88f21a68b21f60412 2024-11-12T06:49:18,187 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a474a06785ead13d749016bf6309b74c/A of a474a06785ead13d749016bf6309b74c into a11c2a6a14374cb88f21a68b21f60412(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:18,187 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:18,187 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., storeName=a474a06785ead13d749016bf6309b74c/A, priority=12, startTime=1731394157325; duration=0sec 2024-11-12T06:49:18,187 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:18,187 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a474a06785ead13d749016bf6309b74c:A 2024-11-12T06:49:18,213 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=604, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/7919aa3987394db6aed73829c777323c 2024-11-12T06:49:18,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/9cbeeae0af494fa8b3e2c16aff27ecb2 is 50, key is test_row_1/B:col10/1731394155703/Put/seqid=0 2024-11-12T06:49:18,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742239_1415 (size=9857) 2024-11-12T06:49:18,629 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=604 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/9cbeeae0af494fa8b3e2c16aff27ecb2 2024-11-12T06:49:18,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/299f4ddfdd094aee8d37d88f8be6dc4d is 50, key is test_row_1/C:col10/1731394155703/Put/seqid=0 2024-11-12T06:49:18,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742240_1416 (size=9857) 2024-11-12T06:49:19,047 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=604 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/299f4ddfdd094aee8d37d88f8be6dc4d 2024-11-12T06:49:19,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/7919aa3987394db6aed73829c777323c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/7919aa3987394db6aed73829c777323c 2024-11-12T06:49:19,065 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/7919aa3987394db6aed73829c777323c, entries=100, sequenceid=604, filesize=22.0 K 2024-11-12T06:49:19,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/9cbeeae0af494fa8b3e2c16aff27ecb2 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/9cbeeae0af494fa8b3e2c16aff27ecb2 2024-11-12T06:49:19,068 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/9cbeeae0af494fa8b3e2c16aff27ecb2, entries=100, sequenceid=604, filesize=9.6 K 2024-11-12T06:49:19,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/299f4ddfdd094aee8d37d88f8be6dc4d as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/299f4ddfdd094aee8d37d88f8be6dc4d 2024-11-12T06:49:19,072 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/299f4ddfdd094aee8d37d88f8be6dc4d, entries=100, sequenceid=604, filesize=9.6 K 2024-11-12T06:49:19,072 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=13.42 KB/13740 for a474a06785ead13d749016bf6309b74c in 1679ms, sequenceid=604, compaction requested=false 2024-11-12T06:49:19,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for a474a06785ead13d749016bf6309b74c: 2024-11-12T06:49:19,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:19,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-12T06:49:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-12T06:49:19,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-12T06:49:19,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1440 sec 2024-11-12T06:49:19,076 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.1470 sec 2024-11-12T06:49:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T06:49:19,087 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:49:22,540 INFO [Thread-1503 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1503. 2024-11-12T06:49:22,540 DEBUG [Thread-1503 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:49:22,540 DEBUG [Thread-1503 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:22,540 DEBUG [Thread-1503 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:22,540 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:49:22,561 INFO [Thread-1497 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1497. 2024-11-12T06:49:22,562 DEBUG [Thread-1497 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:49:22,562 DEBUG [Thread-1497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:22,562 DEBUG [Thread-1497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:22,562 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:49:22,630 INFO [Thread-1501 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1501. 2024-11-12T06:49:22,630 DEBUG [Thread-1501 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:49:22,630 DEBUG [Thread-1501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:22,630 DEBUG [Thread-1501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:22,630 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:49:22,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T06:49:22,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 183 2024-11-12T06:49:22,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-12T06:49:22,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 188 2024-11-12T06:49:22,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-12T06:49:22,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-12T06:49:22,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T06:49:22,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T06:49:22,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1114 2024-11-12T06:49:22,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3337 rows 2024-11-12T06:49:22,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1113 2024-11-12T06:49:22,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3334 rows 2024-11-12T06:49:22,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1118 2024-11-12T06:49:22,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3344 rows 2024-11-12T06:49:22,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1119 2024-11-12T06:49:22,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3352 rows 2024-11-12T06:49:22,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1119 2024-11-12T06:49:22,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3349 rows 2024-11-12T06:49:22,631 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T06:49:22,632 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T06:49:22,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:22,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:22,632 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:49:22,633 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T06:49:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T06:49:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T06:49:22,638 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394162637"}]},"ts":"1731394162637"} 2024-11-12T06:49:22,639 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T06:49:22,639 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T06:49:22,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T06:49:22,641 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a474a06785ead13d749016bf6309b74c, UNASSIGN}] 2024-11-12T06:49:22,641 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a474a06785ead13d749016bf6309b74c, UNASSIGN 2024-11-12T06:49:22,642 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=a474a06785ead13d749016bf6309b74c, regionState=CLOSING, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:22,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a474a06785ead13d749016bf6309b74c, UNASSIGN because future has completed 2024-11-12T06:49:22,643 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T06:49:22,643 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622}] 2024-11-12T06:49:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T06:49:22,795 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:22,795 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T06:49:22,795 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing a474a06785ead13d749016bf6309b74c, disabling compactions & flushes 2024-11-12T06:49:22,796 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:22,796 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:22,796 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. after waiting 0 ms 2024-11-12T06:49:22,796 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:22,796 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing a474a06785ead13d749016bf6309b74c 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-12T06:49:22,796 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=A 2024-11-12T06:49:22,796 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:22,796 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=B 2024-11-12T06:49:22,796 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:22,796 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a474a06785ead13d749016bf6309b74c, store=C 2024-11-12T06:49:22,796 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:22,803 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112630ba5ace9e249faab9da084766bff6f_a474a06785ead13d749016bf6309b74c is 50, key is test_row_0/A:col10/1731394162628/Put/seqid=0 2024-11-12T06:49:22,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742241_1417 (size=9914) 2024-11-12T06:49:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T06:49:23,207 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:23,212 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112630ba5ace9e249faab9da084766bff6f_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112630ba5ace9e249faab9da084766bff6f_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:23,213 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/7b282e43c0294e54964d7555894a2efa, store: [table=TestAcidGuarantees family=A region=a474a06785ead13d749016bf6309b74c] 2024-11-12T06:49:23,213 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/7b282e43c0294e54964d7555894a2efa is 175, key is test_row_0/A:col10/1731394162628/Put/seqid=0 2024-11-12T06:49:23,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742242_1418 (size=22561) 2024-11-12T06:49:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T06:49:23,620 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=615, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/7b282e43c0294e54964d7555894a2efa 2024-11-12T06:49:23,627 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/94aaed54f4ec47b1930b7ef701c83498 is 50, key is test_row_0/B:col10/1731394162628/Put/seqid=0 2024-11-12T06:49:23,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742243_1419 (size=9857) 2024-11-12T06:49:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T06:49:24,031 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/94aaed54f4ec47b1930b7ef701c83498 2024-11-12T06:49:24,043 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/fc2c6cbb6d0a4a86918b358dd513d94e is 50, key is test_row_0/C:col10/1731394162628/Put/seqid=0 2024-11-12T06:49:24,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742244_1420 (size=9857) 2024-11-12T06:49:24,447 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/fc2c6cbb6d0a4a86918b358dd513d94e 2024-11-12T06:49:24,450 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/A/7b282e43c0294e54964d7555894a2efa as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/7b282e43c0294e54964d7555894a2efa 2024-11-12T06:49:24,453 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/7b282e43c0294e54964d7555894a2efa, entries=100, sequenceid=615, filesize=22.0 K 2024-11-12T06:49:24,454 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/B/94aaed54f4ec47b1930b7ef701c83498 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/94aaed54f4ec47b1930b7ef701c83498 2024-11-12T06:49:24,457 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/94aaed54f4ec47b1930b7ef701c83498, entries=100, sequenceid=615, filesize=9.6 K 2024-11-12T06:49:24,458 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/.tmp/C/fc2c6cbb6d0a4a86918b358dd513d94e as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/fc2c6cbb6d0a4a86918b358dd513d94e 2024-11-12T06:49:24,461 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/fc2c6cbb6d0a4a86918b358dd513d94e, entries=100, sequenceid=615, filesize=9.6 K 2024-11-12T06:49:24,462 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a474a06785ead13d749016bf6309b74c in 1666ms, sequenceid=615, compaction requested=true 2024-11-12T06:49:24,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/18b052f3f93c421eab78aa68c380aef8, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/0a01e4feb45e4ea489aa9b0ea81a1e73, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/e60d9b3a27de430ebc47984a88234226, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/2e50e0731e0a415fba6bc7b8205ca50b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/88eeb0011a7c476c8ccd581b1c899686, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/81e4b73c837b4d668064add03a47fb66, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/878ff37d9f1445a38da969c98049eeef, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/07d946caabd1486c94ba17c104c3bf0c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/1b4180e45ad7493380ebc350a12663a7, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/5ac5646bd58f44b2bd96d36627e32694, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/d5c8cd2d96fb4139ba70e5a2ea69865b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/19d42c2b52a548efa6bed6eade339f30, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/34a1ebe265f046eba96f1e4ce0ecd78b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/851d8d37525748279b182e2b7d21fdf6, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/c592171a80e342d3867ad773b529214c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/680629f027ab42fb8a0929a302724aeb, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/9232c617a16d45579fa2ea90a04fa27c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/1a86abfd10154a9e834277706bdefb1f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/53ca41832fef4629b1fb18d3080aa576, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/cd03d61c9ad14c8c85afc468d8d9f101] to archive 2024-11-12T06:49:24,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T06:49:24,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/18b052f3f93c421eab78aa68c380aef8 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/18b052f3f93c421eab78aa68c380aef8 2024-11-12T06:49:24,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/0a01e4feb45e4ea489aa9b0ea81a1e73 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/0a01e4feb45e4ea489aa9b0ea81a1e73 2024-11-12T06:49:24,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/e60d9b3a27de430ebc47984a88234226 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/e60d9b3a27de430ebc47984a88234226 2024-11-12T06:49:24,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/2e50e0731e0a415fba6bc7b8205ca50b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/2e50e0731e0a415fba6bc7b8205ca50b 2024-11-12T06:49:24,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/88eeb0011a7c476c8ccd581b1c899686 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/88eeb0011a7c476c8ccd581b1c899686 2024-11-12T06:49:24,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/81e4b73c837b4d668064add03a47fb66 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/81e4b73c837b4d668064add03a47fb66 2024-11-12T06:49:24,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/878ff37d9f1445a38da969c98049eeef to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/878ff37d9f1445a38da969c98049eeef 2024-11-12T06:49:24,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/07d946caabd1486c94ba17c104c3bf0c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/07d946caabd1486c94ba17c104c3bf0c 2024-11-12T06:49:24,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/1b4180e45ad7493380ebc350a12663a7 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/1b4180e45ad7493380ebc350a12663a7 2024-11-12T06:49:24,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/5ac5646bd58f44b2bd96d36627e32694 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/5ac5646bd58f44b2bd96d36627e32694 2024-11-12T06:49:24,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/d5c8cd2d96fb4139ba70e5a2ea69865b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/d5c8cd2d96fb4139ba70e5a2ea69865b 2024-11-12T06:49:24,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/19d42c2b52a548efa6bed6eade339f30 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/19d42c2b52a548efa6bed6eade339f30 2024-11-12T06:49:24,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/34a1ebe265f046eba96f1e4ce0ecd78b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/34a1ebe265f046eba96f1e4ce0ecd78b 2024-11-12T06:49:24,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/851d8d37525748279b182e2b7d21fdf6 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/851d8d37525748279b182e2b7d21fdf6 2024-11-12T06:49:24,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/c592171a80e342d3867ad773b529214c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/c592171a80e342d3867ad773b529214c 2024-11-12T06:49:24,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/680629f027ab42fb8a0929a302724aeb to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/680629f027ab42fb8a0929a302724aeb 2024-11-12T06:49:24,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/9232c617a16d45579fa2ea90a04fa27c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/9232c617a16d45579fa2ea90a04fa27c 2024-11-12T06:49:24,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/1a86abfd10154a9e834277706bdefb1f to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/1a86abfd10154a9e834277706bdefb1f 2024-11-12T06:49:24,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/53ca41832fef4629b1fb18d3080aa576 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/53ca41832fef4629b1fb18d3080aa576 2024-11-12T06:49:24,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/cd03d61c9ad14c8c85afc468d8d9f101 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/cd03d61c9ad14c8c85afc468d8d9f101 2024-11-12T06:49:24,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/54e89077059c40089b4c5ab678765715, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/9096935fa6b04939acec1209ed541e40, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/ba132d04fdef41b9a7391048b54f9968, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/e8113fac2fa94f9eac85a373b797fde3, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/405ee515f9284c4e816eb670ba6273cf, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/2f2a2e16b9354a1c8c96bfc29663e27c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/81b1223329784193b6a121330f6cade6, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/48b246fc4b77443f8e5bad1b08e84829, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/f7be1a2ba67149c39f0d87d5c7bf0090, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/bc9f7a3b2975487594cfd41994e8d826, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/e78f020042dd4e82b2c108cacc9825bf, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/25b18d0accf04138a5ac9015a6011a7c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/7ab418f86a244d80a460564d96aa309b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/97355ac052fc459fa1826f88060fd8e9, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/94ccfaffb17843fb8b32d20ace6e6efb, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/95e3c62f802f4960874e2b37f125b9b8, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/dfc60af1470c474cb363dd88940c8ac2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/a99f13054f7a4836878c1a7b2e155f92, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/8221bcc8b9d0417489851bbca1f4f246, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/e85cb761c28c49e3ac57810533098243] to archive 2024-11-12T06:49:24,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T06:49:24,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/54e89077059c40089b4c5ab678765715 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/54e89077059c40089b4c5ab678765715 2024-11-12T06:49:24,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/9096935fa6b04939acec1209ed541e40 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/9096935fa6b04939acec1209ed541e40 2024-11-12T06:49:24,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/ba132d04fdef41b9a7391048b54f9968 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/ba132d04fdef41b9a7391048b54f9968 2024-11-12T06:49:24,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/e8113fac2fa94f9eac85a373b797fde3 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/e8113fac2fa94f9eac85a373b797fde3 2024-11-12T06:49:24,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/405ee515f9284c4e816eb670ba6273cf to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/405ee515f9284c4e816eb670ba6273cf 2024-11-12T06:49:24,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/2f2a2e16b9354a1c8c96bfc29663e27c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/2f2a2e16b9354a1c8c96bfc29663e27c 2024-11-12T06:49:24,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/81b1223329784193b6a121330f6cade6 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/81b1223329784193b6a121330f6cade6 2024-11-12T06:49:24,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/48b246fc4b77443f8e5bad1b08e84829 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/48b246fc4b77443f8e5bad1b08e84829 2024-11-12T06:49:24,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/f7be1a2ba67149c39f0d87d5c7bf0090 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/f7be1a2ba67149c39f0d87d5c7bf0090 2024-11-12T06:49:24,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/bc9f7a3b2975487594cfd41994e8d826 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/bc9f7a3b2975487594cfd41994e8d826 2024-11-12T06:49:24,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/e78f020042dd4e82b2c108cacc9825bf to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/e78f020042dd4e82b2c108cacc9825bf 2024-11-12T06:49:24,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/25b18d0accf04138a5ac9015a6011a7c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/25b18d0accf04138a5ac9015a6011a7c 2024-11-12T06:49:24,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/7ab418f86a244d80a460564d96aa309b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/7ab418f86a244d80a460564d96aa309b 2024-11-12T06:49:24,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/97355ac052fc459fa1826f88060fd8e9 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/97355ac052fc459fa1826f88060fd8e9 2024-11-12T06:49:24,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/94ccfaffb17843fb8b32d20ace6e6efb to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/94ccfaffb17843fb8b32d20ace6e6efb 2024-11-12T06:49:24,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/95e3c62f802f4960874e2b37f125b9b8 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/95e3c62f802f4960874e2b37f125b9b8 2024-11-12T06:49:24,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/dfc60af1470c474cb363dd88940c8ac2 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/dfc60af1470c474cb363dd88940c8ac2 2024-11-12T06:49:24,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/a99f13054f7a4836878c1a7b2e155f92 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/a99f13054f7a4836878c1a7b2e155f92 2024-11-12T06:49:24,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/8221bcc8b9d0417489851bbca1f4f246 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/8221bcc8b9d0417489851bbca1f4f246 2024-11-12T06:49:24,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/e85cb761c28c49e3ac57810533098243 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/e85cb761c28c49e3ac57810533098243 2024-11-12T06:49:24,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/630728e4b6bb48a3bcc21dbe7c0a8f85, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/798a8a30c13044e7beb88e07f7ec50fa, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/7ec464ff4b0a4315a07bd4b7831ade88, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/4953e493bd5b47ceb46fd651f5a47a45, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/6482cb4b6d5d4dc4b29ac9094fb43688, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/3d2060d290de440091ae84dec9f5c3a9, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/6fdabf840350465280294d393a67f0bb, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/62e7696abf994f41a8b5612fb4f03218, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/6e3709f4cb92459bacfe2fc42b303f1b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/2c29106e6b904125bb59502a019803f5, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/9ecbef4dfa2e4ebe80982c26b92eb7e6, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/176a25f7ec134d81a2704498286d692b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/d017c25c7f204ae9bf997dc348e39a0e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/c61c7bee36364a05ac443bece86464d8, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/fdcd7400a5ab47e89e8a689f5d1d5ca4, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/dd0b1c5a52fb4da3a4e4222cd281a578, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/d3367f0920a040138568ae6bb67b9a6c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/443f7c40e13f4ad285c95b85ce2826f5, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/ea6ffce713ce4ec4945cc44198438203, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/87b0645e119e4394a5da8434a9f4234b] to archive 2024-11-12T06:49:24,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T06:49:24,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/630728e4b6bb48a3bcc21dbe7c0a8f85 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/630728e4b6bb48a3bcc21dbe7c0a8f85 2024-11-12T06:49:24,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/798a8a30c13044e7beb88e07f7ec50fa to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/798a8a30c13044e7beb88e07f7ec50fa 2024-11-12T06:49:24,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/7ec464ff4b0a4315a07bd4b7831ade88 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/7ec464ff4b0a4315a07bd4b7831ade88 2024-11-12T06:49:24,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/4953e493bd5b47ceb46fd651f5a47a45 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/4953e493bd5b47ceb46fd651f5a47a45 2024-11-12T06:49:24,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/6482cb4b6d5d4dc4b29ac9094fb43688 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/6482cb4b6d5d4dc4b29ac9094fb43688 2024-11-12T06:49:24,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/3d2060d290de440091ae84dec9f5c3a9 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/3d2060d290de440091ae84dec9f5c3a9 2024-11-12T06:49:24,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/6fdabf840350465280294d393a67f0bb to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/6fdabf840350465280294d393a67f0bb 2024-11-12T06:49:24,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/62e7696abf994f41a8b5612fb4f03218 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/62e7696abf994f41a8b5612fb4f03218 2024-11-12T06:49:24,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/6e3709f4cb92459bacfe2fc42b303f1b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/6e3709f4cb92459bacfe2fc42b303f1b 2024-11-12T06:49:24,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/2c29106e6b904125bb59502a019803f5 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/2c29106e6b904125bb59502a019803f5 2024-11-12T06:49:24,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/9ecbef4dfa2e4ebe80982c26b92eb7e6 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/9ecbef4dfa2e4ebe80982c26b92eb7e6 2024-11-12T06:49:24,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/176a25f7ec134d81a2704498286d692b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/176a25f7ec134d81a2704498286d692b 2024-11-12T06:49:24,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/d017c25c7f204ae9bf997dc348e39a0e to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/d017c25c7f204ae9bf997dc348e39a0e 2024-11-12T06:49:24,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/c61c7bee36364a05ac443bece86464d8 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/c61c7bee36364a05ac443bece86464d8 2024-11-12T06:49:24,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/fdcd7400a5ab47e89e8a689f5d1d5ca4 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/fdcd7400a5ab47e89e8a689f5d1d5ca4 2024-11-12T06:49:24,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/dd0b1c5a52fb4da3a4e4222cd281a578 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/dd0b1c5a52fb4da3a4e4222cd281a578 2024-11-12T06:49:24,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/d3367f0920a040138568ae6bb67b9a6c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/d3367f0920a040138568ae6bb67b9a6c 2024-11-12T06:49:24,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/443f7c40e13f4ad285c95b85ce2826f5 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/443f7c40e13f4ad285c95b85ce2826f5 2024-11-12T06:49:24,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/ea6ffce713ce4ec4945cc44198438203 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/ea6ffce713ce4ec4945cc44198438203 2024-11-12T06:49:24,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/87b0645e119e4394a5da8434a9f4234b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/87b0645e119e4394a5da8434a9f4234b 2024-11-12T06:49:24,517 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/recovered.edits/618.seqid, newMaxSeqId=618, maxSeqId=4 2024-11-12T06:49:24,517 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. 2024-11-12T06:49:24,517 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for a474a06785ead13d749016bf6309b74c: Waiting for close lock at 1731394162795Running coprocessor pre-close hooks at 1731394162795Disabling compacts and flushes for region at 1731394162795Disabling writes for close at 1731394162796 (+1 ms)Obtaining lock to block concurrent updates at 1731394162796Preparing flush snapshotting stores in a474a06785ead13d749016bf6309b74c at 1731394162796Finished memstore snapshotting TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731394162796Flushing stores of TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c. at 1731394162797 (+1 ms)Flushing a474a06785ead13d749016bf6309b74c/A: creating writer at 1731394162797Flushing mob file a474a06785ead13d749016bf6309b74c/A: appending metadata at 1731394162803 (+6 ms)Flushing mob file a474a06785ead13d749016bf6309b74c/A: closing flushed file at 1731394162803Flushing a474a06785ead13d749016bf6309b74c/A: appending metadata at 1731394163213 (+410 ms)Flushing a474a06785ead13d749016bf6309b74c/A: closing flushed file at 1731394163213Flushing a474a06785ead13d749016bf6309b74c/B: creating writer at 1731394163624 (+411 ms)Flushing a474a06785ead13d749016bf6309b74c/B: appending metadata at 1731394163627 (+3 ms)Flushing a474a06785ead13d749016bf6309b74c/B: closing flushed file at 1731394163627Flushing a474a06785ead13d749016bf6309b74c/C: creating writer at 1731394164037 (+410 ms)Flushing a474a06785ead13d749016bf6309b74c/C: appending metadata at 1731394164042 (+5 ms)Flushing a474a06785ead13d749016bf6309b74c/C: closing flushed file at 1731394164042Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15284c08: reopening flushed file at 1731394164450 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c434bdd: reopening flushed file at 1731394164453 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63ce2cfa: reopening flushed file at 1731394164457 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a474a06785ead13d749016bf6309b74c in 1666ms, sequenceid=615, compaction requested=true at 1731394164462 (+5 ms)Writing region close event to WAL at 1731394164514 (+52 ms)Running coprocessor post-close hooks at 1731394164517 (+3 ms)Closed at 1731394164517 2024-11-12T06:49:24,520 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:24,520 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=a474a06785ead13d749016bf6309b74c, regionState=CLOSED 2024-11-12T06:49:24,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 because future has completed 2024-11-12T06:49:24,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-12T06:49:24,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure a474a06785ead13d749016bf6309b74c, server=9f0beee3d6b6,38777,1731394047622 in 1.8790 sec 2024-11-12T06:49:24,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-12T06:49:24,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a474a06785ead13d749016bf6309b74c, UNASSIGN in 1.8830 sec 2024-11-12T06:49:24,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-12T06:49:24,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8850 sec 2024-11-12T06:49:24,528 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394164527"}]},"ts":"1731394164527"} 2024-11-12T06:49:24,529 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T06:49:24,529 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T06:49:24,530 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8960 sec 2024-11-12T06:49:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T06:49:24,788 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:49:24,788 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T06:49:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:49:24,790 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:49:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-12T06:49:24,792 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:49:24,795 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:24,799 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A, FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B, FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C, FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/recovered.edits] 2024-11-12T06:49:24,803 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/7919aa3987394db6aed73829c777323c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/7919aa3987394db6aed73829c777323c 2024-11-12T06:49:24,803 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/7b282e43c0294e54964d7555894a2efa to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/7b282e43c0294e54964d7555894a2efa 2024-11-12T06:49:24,804 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/a11c2a6a14374cb88f21a68b21f60412 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/A/a11c2a6a14374cb88f21a68b21f60412 2024-11-12T06:49:24,806 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/89c96c2a940e4a9cb7a79a5c1e2f4a73 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/89c96c2a940e4a9cb7a79a5c1e2f4a73 2024-11-12T06:49:24,807 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/94aaed54f4ec47b1930b7ef701c83498 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/94aaed54f4ec47b1930b7ef701c83498 2024-11-12T06:49:24,808 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/9cbeeae0af494fa8b3e2c16aff27ecb2 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/B/9cbeeae0af494fa8b3e2c16aff27ecb2 2024-11-12T06:49:24,810 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/12f1b8f90f4c4df2a111fd2d06dd61e6 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/12f1b8f90f4c4df2a111fd2d06dd61e6 2024-11-12T06:49:24,810 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/299f4ddfdd094aee8d37d88f8be6dc4d to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/299f4ddfdd094aee8d37d88f8be6dc4d 2024-11-12T06:49:24,811 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/fc2c6cbb6d0a4a86918b358dd513d94e to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/C/fc2c6cbb6d0a4a86918b358dd513d94e 2024-11-12T06:49:24,813 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/recovered.edits/618.seqid to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c/recovered.edits/618.seqid 2024-11-12T06:49:24,813 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:24,813 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T06:49:24,814 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T06:49:24,814 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-12T06:49:24,816 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112289122a1125543119098c558dc03a2a5_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112289122a1125543119098c558dc03a2a5_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:24,817 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111232610c2b1a1b4a04b7b868870153a552_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111232610c2b1a1b4a04b7b868870153a552_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:24,818 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111237a8963f9fea4713be57120239cbc237_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111237a8963f9fea4713be57120239cbc237_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:24,818 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123efd11a368ad43da8d1e7084ba2b0184_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123efd11a368ad43da8d1e7084ba2b0184_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:24,819 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112450caf6ff0cc458d850299ab5eb6f19a_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112450caf6ff0cc458d850299ab5eb6f19a_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:24,820 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124a19ccb20d1a4b27a60f74d890de79a8_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124a19ccb20d1a4b27a60f74d890de79a8_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:24,821 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124eb67913be504945993455a4b820b87a_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124eb67913be504945993455a4b820b87a_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:24,821 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111251d15dfdeb0c4cd88586e9bb05a9fdaa_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111251d15dfdeb0c4cd88586e9bb05a9fdaa_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:24,822 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112531d33f469f74bfea546ed2fbda4d59c_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112531d33f469f74bfea546ed2fbda4d59c_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:24,823 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112630ba5ace9e249faab9da084766bff6f_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112630ba5ace9e249faab9da084766bff6f_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:24,824 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126de05bb87e3d49b39bf805e580772b14_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126de05bb87e3d49b39bf805e580772b14_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:24,824 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127b3cb020cf2c444bb3e2f5e559aa25b1_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127b3cb020cf2c444bb3e2f5e559aa25b1_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:24,825 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127cb4bfaec12e4c2aa106960f12694138_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127cb4bfaec12e4c2aa106960f12694138_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:24,826 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128d323f2dccea438f87a2c68292219739_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128d323f2dccea438f87a2c68292219739_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:24,826 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128eb5a61ee8d44c98ae88d6ccb31d80c2_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128eb5a61ee8d44c98ae88d6ccb31d80c2_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:24,827 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128f2e6cc8001846b8874e2d354ff536d4_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128f2e6cc8001846b8874e2d354ff536d4_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:24,828 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d416c987bc484b6eac84260d6933406d_a474a06785ead13d749016bf6309b74c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d416c987bc484b6eac84260d6933406d_a474a06785ead13d749016bf6309b74c 2024-11-12T06:49:24,828 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T06:49:24,830 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:49:24,831 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T06:49:24,833 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T06:49:24,834 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:49:24,834 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T06:49:24,834 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731394164834"}]},"ts":"9223372036854775807"} 2024-11-12T06:49:24,836 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T06:49:24,836 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a474a06785ead13d749016bf6309b74c, NAME => 'TestAcidGuarantees,,1731394133824.a474a06785ead13d749016bf6309b74c.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T06:49:24,836 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T06:49:24,836 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731394164836"}]},"ts":"9223372036854775807"} 2024-11-12T06:49:24,838 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T06:49:24,839 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:49:24,840 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-11-12T06:49:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-12T06:49:24,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T06:49:24,898 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:49:24,908 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 251), OpenFileDescriptor=451 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=333 (was 348), ProcessCount=11 (was 11), AvailableMemoryMB=6567 (was 6654) 2024-11-12T06:49:24,918 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=333, ProcessCount=11, AvailableMemoryMB=6567 2024-11-12T06:49:24,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T06:49:24,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T06:49:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T06:49:24,921 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T06:49:24,921 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:24,921 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-12T06:49:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-12T06:49:24,922 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T06:49:24,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742245_1421 (size=1061) 2024-11-12T06:49:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-12T06:49:25,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-12T06:49:25,330 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e 2024-11-12T06:49:25,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742246_1422 (size=53) 2024-11-12T06:49:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-12T06:49:25,738 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T06:49:25,738 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing efaada0e985d01f31ce286661347290d, disabling compactions & flushes 2024-11-12T06:49:25,738 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:25,738 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:25,738 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. after waiting 0 ms 2024-11-12T06:49:25,739 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:25,739 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:25,739 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for efaada0e985d01f31ce286661347290d: Waiting for close lock at 1731394165738Disabling compacts and flushes for region at 1731394165738Disabling writes for close at 1731394165739 (+1 ms)Writing region close event to WAL at 1731394165739Closed at 1731394165739 2024-11-12T06:49:25,741 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T06:49:25,742 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731394165741"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731394165741"}]},"ts":"1731394165741"} 2024-11-12T06:49:25,746 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T06:49:25,747 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T06:49:25,747 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394165747"}]},"ts":"1731394165747"} 2024-11-12T06:49:25,748 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T06:49:25,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efaada0e985d01f31ce286661347290d, ASSIGN}] 2024-11-12T06:49:25,749 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efaada0e985d01f31ce286661347290d, ASSIGN 2024-11-12T06:49:25,750 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=efaada0e985d01f31ce286661347290d, ASSIGN; state=OFFLINE, location=9f0beee3d6b6,38777,1731394047622; forceNewPlan=false, retain=false 2024-11-12T06:49:25,901 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=efaada0e985d01f31ce286661347290d, regionState=OPENING, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:25,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=efaada0e985d01f31ce286661347290d, ASSIGN because future has completed 2024-11-12T06:49:25,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622}] 2024-11-12T06:49:25,969 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T06:49:26,057 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:26,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-12T06:49:26,057 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} 2024-11-12T06:49:26,058 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees efaada0e985d01f31ce286661347290d 2024-11-12T06:49:26,058 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T06:49:26,058 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for efaada0e985d01f31ce286661347290d 2024-11-12T06:49:26,058 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for efaada0e985d01f31ce286661347290d 2024-11-12T06:49:26,060 INFO [StoreOpener-efaada0e985d01f31ce286661347290d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region efaada0e985d01f31ce286661347290d 2024-11-12T06:49:26,061 INFO [StoreOpener-efaada0e985d01f31ce286661347290d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:49:26,062 INFO [StoreOpener-efaada0e985d01f31ce286661347290d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efaada0e985d01f31ce286661347290d columnFamilyName A 2024-11-12T06:49:26,062 DEBUG [StoreOpener-efaada0e985d01f31ce286661347290d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:26,062 INFO [StoreOpener-efaada0e985d01f31ce286661347290d-1 {}] regionserver.HStore(327): Store=efaada0e985d01f31ce286661347290d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:49:26,063 INFO [StoreOpener-efaada0e985d01f31ce286661347290d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region efaada0e985d01f31ce286661347290d 2024-11-12T06:49:26,064 INFO [StoreOpener-efaada0e985d01f31ce286661347290d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:49:26,064 INFO [StoreOpener-efaada0e985d01f31ce286661347290d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efaada0e985d01f31ce286661347290d columnFamilyName B 2024-11-12T06:49:26,064 DEBUG [StoreOpener-efaada0e985d01f31ce286661347290d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:26,065 INFO [StoreOpener-efaada0e985d01f31ce286661347290d-1 {}] regionserver.HStore(327): Store=efaada0e985d01f31ce286661347290d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:49:26,065 INFO [StoreOpener-efaada0e985d01f31ce286661347290d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region efaada0e985d01f31ce286661347290d 2024-11-12T06:49:26,066 INFO [StoreOpener-efaada0e985d01f31ce286661347290d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:49:26,066 INFO [StoreOpener-efaada0e985d01f31ce286661347290d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efaada0e985d01f31ce286661347290d columnFamilyName C 2024-11-12T06:49:26,066 DEBUG [StoreOpener-efaada0e985d01f31ce286661347290d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:26,066 INFO [StoreOpener-efaada0e985d01f31ce286661347290d-1 {}] regionserver.HStore(327): Store=efaada0e985d01f31ce286661347290d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:49:26,067 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:26,067 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for efaada0e985d01f31ce286661347290d 2024-11-12T06:49:26,067 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d 2024-11-12T06:49:26,068 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d 2024-11-12T06:49:26,069 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for efaada0e985d01f31ce286661347290d 2024-11-12T06:49:26,069 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for efaada0e985d01f31ce286661347290d 2024-11-12T06:49:26,069 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T06:49:26,070 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for efaada0e985d01f31ce286661347290d 2024-11-12T06:49:26,072 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T06:49:26,072 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened efaada0e985d01f31ce286661347290d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72598757, jitterRate=0.08180578052997589}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T06:49:26,072 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for efaada0e985d01f31ce286661347290d 2024-11-12T06:49:26,073 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for efaada0e985d01f31ce286661347290d: Running coprocessor pre-open hook at 1731394166058Writing region info on filesystem at 1731394166058Initializing all the Stores at 1731394166059 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394166059Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394166060 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394166060Cleaning up temporary data from old regions at 1731394166069 (+9 ms)Running coprocessor post-open hooks at 1731394166072 (+3 ms)Region opened successfully at 1731394166073 (+1 ms) 2024-11-12T06:49:26,073 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., pid=118, masterSystemTime=1731394166054 2024-11-12T06:49:26,075 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:26,075 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:26,075 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=efaada0e985d01f31ce286661347290d, regionState=OPEN, openSeqNum=2, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:26,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 because future has completed 2024-11-12T06:49:26,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-12T06:49:26,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 in 175 msec 2024-11-12T06:49:26,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-12T06:49:26,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efaada0e985d01f31ce286661347290d, ASSIGN in 330 msec 2024-11-12T06:49:26,080 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T06:49:26,080 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394166080"}]},"ts":"1731394166080"} 2024-11-12T06:49:26,081 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T06:49:26,082 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T06:49:26,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-11-12T06:49:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-12T06:49:27,068 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:49:27,068 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8abc8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,069 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:49:27,069 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:49:27,069 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:49:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:49:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:49:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d7badd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:49:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:49:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:27,071 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:49:27,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73970e0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,071 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:49:27,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:49:27,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:49:27,073 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60212, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:49:27,074 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T06:49:27,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 9f0beee3d6b6,43031,1731394046864 2024-11-12T06:49:27,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7b24502d 2024-11-12T06:49:27,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T06:49:27,075 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T06:49:27,076 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bd122f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,076 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:49:27,076 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:49:27,077 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:49:27,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:49:27,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:49:27,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4329135b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:49:27,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:49:27,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:27,078 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:49:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@417e425f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,079 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:49:27,079 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:49:27,079 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:49:27,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:49:27,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:49:27,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4447b022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:49:27,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:49:27,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:27,080 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:49:27,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@759bcbfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,081 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,081 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:49:27,082 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:49:27,082 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:49:27,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:49:27,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:49:27,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3063015a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:49:27,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:49:27,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:27,083 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:49:27,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@644aff7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,084 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,084 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:49:27,084 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:49:27,085 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:49:27,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:49:27,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:49:27,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5826a55f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:49:27,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:49:27,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:27,086 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:49:27,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@268e3447, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,087 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,087 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:49:27,087 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:49:27,087 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:49:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:49:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:49:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6be4f1b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:49:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:49:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:27,088 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:49:27,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8bea1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,089 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:49:27,089 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:49:27,090 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:49:27,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:49:27,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:49:27,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27060dc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:49:27,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:49:27,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:27,091 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:49:27,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5df6258a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,092 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,092 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:49:27,092 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:49:27,092 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:49:27,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:49:27,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:49:27,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de5c972, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:49:27,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:49:27,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:27,093 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:49:27,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2401f678, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,094 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:49:27,094 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:49:27,095 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:49:27,095 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:49:27,095 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:49:27,095 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3728be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,095 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:49:27,096 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:49:27,096 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:27,096 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:49:27,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a0b2c8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:49:27,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:49:27,098 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:49:27,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:49:27,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:49:27,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ab9aaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:49:27,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:49:27,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:27,099 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:49:27,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b44268b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,100 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,100 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:49:27,100 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:49:27,100 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:49:27,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:49:27,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:49:27,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c634126, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:49:27,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:49:27,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:27,101 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:49:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77dcc433, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:27,102 DEBUG [Thread-1903 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:49:27,102 DEBUG [Thread-1918 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:49:27,103 DEBUG [Thread-1907 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:49:27,103 DEBUG [Thread-1912 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:49:27,103 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:49:27,103 DEBUG [Thread-1905 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:49:27,103 DEBUG [Thread-1920 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:49:27,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:49:27,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:49:27,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:49:27,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:49:27,103 DEBUG [Thread-1901 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:49:27,103 DEBUG [Thread-1909 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:49:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:49:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:49:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:49:27,104 DEBUG [Thread-1916 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:49:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:49:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:49:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:49:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:49:27,105 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:49:27,105 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:49:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:49:27,105 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:49:27,105 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:49:27,105 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:49:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:49:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:49:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:49:27,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:49:27,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:49:27,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:49:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-12T06:49:27,106 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:49:27,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:49:27,107 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:49:27,107 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:49:27,107 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:49:27,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:49:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T06:49:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:49:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:49:27,109 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:49:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:49:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:49:27,111 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:49:27,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:49:27,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:49:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:49:27,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:49:27,113 DEBUG [Thread-1914 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:49:27,114 DEBUG [Thread-1914 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:49:27,115 DEBUG [Thread-1914 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:49:27,116 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:49:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on efaada0e985d01f31ce286661347290d 2024-11-12T06:49:27,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efaada0e985d01f31ce286661347290d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T06:49:27,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=A 2024-11-12T06:49:27,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:27,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=B 2024-11-12T06:49:27,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:27,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=C 2024-11-12T06:49:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2] 2024-11-12T06:49:27,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:27,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/6bfcb3bfb3734c9a812bab23618e4edf is 50, key is test_row_0/A:col10/1731394167113/Put/seqid=0 2024-11-12T06:49:27,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731394167127, max timestamp=1731394167161Num uniques 859; 2024-11-12T06:49:27,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T06:49:27,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731394167127, max timestamp=1731394167161Num uniques 859; 2024-11-12T06:49:27,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T06:49:27,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731394167127, max timestamp=1731394167161Num uniques 859; 2024-11-12T06:49:27,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T06:49:27,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742247_1423 (size=9657) 2024-11-12T06:49:27,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394227195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:27,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:27,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394227197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:27,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T06:49:27,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394227210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:27,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:27,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394227212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:27,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:27,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:27,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394227213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:27,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-12T06:49:27,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:27,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:27,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:27,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:27,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:27,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394227317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394227317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:27,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394227327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:27,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:27,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394227328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:27,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:27,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394227337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:27,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:27,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-12T06:49:27,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:27,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:27,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:27,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:27,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:27,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T06:49:27,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394227527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:27,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:27,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394227528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:27,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:27,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:27,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:27,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394227537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:27,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:27,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394227537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:27,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:27,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:27,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394227557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:27,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:27,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-12T06:49:27,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:27,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:27,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:27,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:27,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:27,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:27,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/6bfcb3bfb3734c9a812bab23618e4edf 2024-11-12T06:49:27,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/26bb813b231c4668be17ab531bc4007e is 50, key is test_row_0/B:col10/1731394167113/Put/seqid=0 2024-11-12T06:49:27,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742248_1424 (size=9657) 2024-11-12T06:49:27,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-12T06:49:27,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:27,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:27,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:27,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:27,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:27,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T06:49:27,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394227837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:27,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:27,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394227847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:27,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:27,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394227847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:27,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:27,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:27,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394227847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:27,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:27,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:27,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394227867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:27,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:27,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:27,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-12T06:49:27,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:27,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:27,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:27,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:27,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:28,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/26bb813b231c4668be17ab531bc4007e 2024-11-12T06:49:28,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-12T06:49:28,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:28,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:28,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:28,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:28,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:28,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/e3750f06387442d1a4000dca780de1f1 is 50, key is test_row_0/C:col10/1731394167113/Put/seqid=0 2024-11-12T06:49:28,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742249_1425 (size=9657) 2024-11-12T06:49:28,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-12T06:49:28,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:28,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:28,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:28,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:28,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:28,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T06:49:28,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-12T06:49:28,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:28,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:28,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:28,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:28,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:28,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394228347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:28,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:28,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:28,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:28,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394228357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:28,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394228357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:28,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394228357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:28,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:28,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:28,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:28,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394228377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:28,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:28,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:28,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:28,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/e3750f06387442d1a4000dca780de1f1 2024-11-12T06:49:28,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/6bfcb3bfb3734c9a812bab23618e4edf as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/6bfcb3bfb3734c9a812bab23618e4edf 2024-11-12T06:49:28,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/6bfcb3bfb3734c9a812bab23618e4edf, entries=100, sequenceid=11, filesize=9.4 K 2024-11-12T06:49:28,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/26bb813b231c4668be17ab531bc4007e as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/26bb813b231c4668be17ab531bc4007e 2024-11-12T06:49:28,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/26bb813b231c4668be17ab531bc4007e, entries=100, sequenceid=11, filesize=9.4 K 2024-11-12T06:49:28,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/e3750f06387442d1a4000dca780de1f1 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/e3750f06387442d1a4000dca780de1f1 2024-11-12T06:49:28,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/e3750f06387442d1a4000dca780de1f1, entries=100, sequenceid=11, filesize=9.4 K 2024-11-12T06:49:28,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for efaada0e985d01f31ce286661347290d in 1361ms, sequenceid=11, compaction requested=false 2024-11-12T06:49:28,479 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-12T06:49:28,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:28,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-12T06:49:28,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:28,492 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing efaada0e985d01f31ce286661347290d 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-12T06:49:28,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=A 2024-11-12T06:49:28,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:28,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=B 2024-11-12T06:49:28,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:28,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=C 2024-11-12T06:49:28,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:28,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/a13016a9637143a5afa4a51adce2fc4b is 50, key is test_row_0/A:col10/1731394167210/Put/seqid=0 2024-11-12T06:49:28,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742250_1426 (size=12001) 2024-11-12T06:49:28,909 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/a13016a9637143a5afa4a51adce2fc4b 2024-11-12T06:49:28,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/06609cc4ea1d44029d0739a386aad6c2 is 50, key is test_row_0/B:col10/1731394167210/Put/seqid=0 2024-11-12T06:49:28,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742251_1427 (size=12001) 2024-11-12T06:49:28,924 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/06609cc4ea1d44029d0739a386aad6c2 2024-11-12T06:49:28,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/06775c328594473bbb4287a64868bb81 is 50, key is test_row_0/C:col10/1731394167210/Put/seqid=0 2024-11-12T06:49:28,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742252_1428 (size=12001) 2024-11-12T06:49:28,946 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/06775c328594473bbb4287a64868bb81 2024-11-12T06:49:28,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/a13016a9637143a5afa4a51adce2fc4b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/a13016a9637143a5afa4a51adce2fc4b 2024-11-12T06:49:28,962 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/a13016a9637143a5afa4a51adce2fc4b, entries=150, sequenceid=46, filesize=11.7 K 2024-11-12T06:49:28,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/06609cc4ea1d44029d0739a386aad6c2 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/06609cc4ea1d44029d0739a386aad6c2 2024-11-12T06:49:28,971 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/06609cc4ea1d44029d0739a386aad6c2, entries=150, sequenceid=46, filesize=11.7 K 2024-11-12T06:49:28,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/06775c328594473bbb4287a64868bb81 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/06775c328594473bbb4287a64868bb81 2024-11-12T06:49:28,975 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/06775c328594473bbb4287a64868bb81, entries=150, sequenceid=46, filesize=11.7 K 2024-11-12T06:49:28,976 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=0 B/0 for efaada0e985d01f31ce286661347290d in 484ms, sequenceid=46, compaction requested=false 2024-11-12T06:49:28,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:28,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:28,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-12T06:49:28,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-12T06:49:28,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-12T06:49:28,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8670 sec 2024-11-12T06:49:28,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.8770 sec 2024-11-12T06:49:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T06:49:29,257 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:49:29,257 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:49:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-12T06:49:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T06:49:29,259 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:49:29,260 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:49:29,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:49:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T06:49:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on efaada0e985d01f31ce286661347290d 2024-11-12T06:49:29,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efaada0e985d01f31ce286661347290d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T06:49:29,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=A 2024-11-12T06:49:29,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:29,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=B 2024-11-12T06:49:29,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:29,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=C 2024-11-12T06:49:29,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:29,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/3eeb8796b18043309faa71bd9f0942f7 is 50, key is test_row_0/A:col10/1731394169366/Put/seqid=0 2024-11-12T06:49:29,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742253_1429 (size=16681) 2024-11-12T06:49:29,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T06:49:29,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:29,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:29,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:29,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:29,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:29,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394169405, max timestamp=1731394169427Num uniques 859; 2024-11-12T06:49:29,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394169405, max timestamp=1731394169427Num uniques 859; 2024-11-12T06:49:29,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394169405, max timestamp=1731394169427Num uniques 859; 2024-11-12T06:49:29,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:29,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:29,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:29,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394229456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:29,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394229462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:29,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:29,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:29,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394229499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:29,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394229499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:29,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394229499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:29,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:29,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T06:49:29,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:29,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:29,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:29,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:29,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T06:49:29,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394229606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:29,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:29,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394229607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:29,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:29,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:29,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394229627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:29,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:29,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394229627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:29,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:29,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394229629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:29,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T06:49:29,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:29,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:29,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:29,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:29,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:29,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/3eeb8796b18043309faa71bd9f0942f7 2024-11-12T06:49:29,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/7602a8f0aa0b4c699c61a29d212c4485 is 50, key is test_row_0/B:col10/1731394169366/Put/seqid=0 2024-11-12T06:49:29,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742254_1430 (size=12001) 2024-11-12T06:49:29,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394229827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:29,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394229827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:29,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:29,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394229837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:29,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:29,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394229837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:29,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:29,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:29,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394229837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:29,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:29,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:29,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:29,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T06:49:29,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:29,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:29,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:29,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:29,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:29,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T06:49:30,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T06:49:30,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:30,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:30,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:30,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:30,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:30,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:30,035 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T06:49:30,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:30,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394230147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:30,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394230147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:30,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:30,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:30,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:30,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394230157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:30,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:30,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394230157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:30,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:30,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:30,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:30,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394230157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:30,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:30,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T06:49:30,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:30,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:30,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:30,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:30,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:30,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:30,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/7602a8f0aa0b4c699c61a29d212c4485 2024-11-12T06:49:30,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/08f57a6a77924fcb82b40c8436fd1d56 is 50, key is test_row_0/C:col10/1731394169366/Put/seqid=0 2024-11-12T06:49:30,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742255_1431 (size=12001) 2024-11-12T06:49:30,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T06:49:30,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:30,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:30,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:30,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:30,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:30,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T06:49:30,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T06:49:30,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:30,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:30,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:30,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:30,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:30,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/08f57a6a77924fcb82b40c8436fd1d56 2024-11-12T06:49:30,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/3eeb8796b18043309faa71bd9f0942f7 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/3eeb8796b18043309faa71bd9f0942f7 2024-11-12T06:49:30,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/3eeb8796b18043309faa71bd9f0942f7, entries=250, sequenceid=56, filesize=16.3 K 2024-11-12T06:49:30,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/7602a8f0aa0b4c699c61a29d212c4485 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/7602a8f0aa0b4c699c61a29d212c4485 2024-11-12T06:49:30,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T06:49:30,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:30,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:30,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:30,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:30,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:30,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/7602a8f0aa0b4c699c61a29d212c4485, entries=150, sequenceid=56, filesize=11.7 K 2024-11-12T06:49:30,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/08f57a6a77924fcb82b40c8436fd1d56 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/08f57a6a77924fcb82b40c8436fd1d56 2024-11-12T06:49:30,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/08f57a6a77924fcb82b40c8436fd1d56, entries=150, sequenceid=56, filesize=11.7 K 2024-11-12T06:49:30,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for efaada0e985d01f31ce286661347290d in 1268ms, sequenceid=56, compaction requested=true 2024-11-12T06:49:30,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:30,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:49:30,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:30,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:49:30,647 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:30,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:30,647 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:30,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:49:30,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:30,648 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:30,648 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:30,648 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/A is initiating minor compaction (all files) 2024-11-12T06:49:30,648 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/B is initiating minor compaction (all files) 2024-11-12T06:49:30,648 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/A in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:30,648 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/B in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:30,648 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/6bfcb3bfb3734c9a812bab23618e4edf, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/a13016a9637143a5afa4a51adce2fc4b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/3eeb8796b18043309faa71bd9f0942f7] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=37.4 K 2024-11-12T06:49:30,648 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/26bb813b231c4668be17ab531bc4007e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/06609cc4ea1d44029d0739a386aad6c2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/7602a8f0aa0b4c699c61a29d212c4485] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=32.9 K 2024-11-12T06:49:30,648 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bfcb3bfb3734c9a812bab23618e4edf, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731394167113 2024-11-12T06:49:30,648 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26bb813b231c4668be17ab531bc4007e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731394167113 2024-11-12T06:49:30,648 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting a13016a9637143a5afa4a51adce2fc4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731394167208 2024-11-12T06:49:30,648 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06609cc4ea1d44029d0739a386aad6c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731394167208 2024-11-12T06:49:30,648 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eeb8796b18043309faa71bd9f0942f7, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731394169361 2024-11-12T06:49:30,648 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7602a8f0aa0b4c699c61a29d212c4485, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731394169366 2024-11-12T06:49:30,654 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#B#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:30,654 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/0149efffb6f84f8695ba9047e8935bac is 50, key is test_row_0/B:col10/1731394169366/Put/seqid=0 2024-11-12T06:49:30,659 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#A#compaction#355 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:30,659 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/0f7a8b4b4902472b902c30638a711d6c is 50, key is test_row_0/A:col10/1731394169366/Put/seqid=0 2024-11-12T06:49:30,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742257_1433 (size=12104) 2024-11-12T06:49:30,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742256_1432 (size=12104) 2024-11-12T06:49:30,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on efaada0e985d01f31ce286661347290d 2024-11-12T06:49:30,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efaada0e985d01f31ce286661347290d 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-12T06:49:30,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=A 2024-11-12T06:49:30,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:49:30,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=B 2024-11-12T06:49:30,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:49:30,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=C 2024-11-12T06:49:30,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:49:30,678 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/0f7a8b4b4902472b902c30638a711d6c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/0f7a8b4b4902472b902c30638a711d6c 2024-11-12T06:49:30,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/91e216b1083649a69d81f5cde3d0bccd is 50, key is test_row_0/A:col10/1731394170668/Put/seqid=0 2024-11-12T06:49:30,681 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efaada0e985d01f31ce286661347290d/A of efaada0e985d01f31ce286661347290d into 0f7a8b4b4902472b902c30638a711d6c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:30,681 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:30,681 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/A, priority=13, startTime=1731394170647; duration=0sec 2024-11-12T06:49:30,681 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:30,681 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:A 2024-11-12T06:49:30,681 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:30,682 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:30,682 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/C is initiating minor compaction (all files) 2024-11-12T06:49:30,682 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/C in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:30,682 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/e3750f06387442d1a4000dca780de1f1, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/06775c328594473bbb4287a64868bb81, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/08f57a6a77924fcb82b40c8436fd1d56] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=32.9 K 2024-11-12T06:49:30,682 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting e3750f06387442d1a4000dca780de1f1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731394167113 2024-11-12T06:49:30,683 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 06775c328594473bbb4287a64868bb81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731394167208 2024-11-12T06:49:30,683 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 08f57a6a77924fcb82b40c8436fd1d56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731394169366 2024-11-12T06:49:30,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742258_1434 (size=14341) 2024-11-12T06:49:30,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/91e216b1083649a69d81f5cde3d0bccd 2024-11-12T06:49:30,690 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#C#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:30,691 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/59dacdbadb8e42f3be9a07debff7770f is 50, key is test_row_0/C:col10/1731394169366/Put/seqid=0 2024-11-12T06:49:30,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394230686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:30,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/7455e0c16f734e3e8845bea3251689fa is 50, key is test_row_0/B:col10/1731394170668/Put/seqid=0 2024-11-12T06:49:30,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:30,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742259_1435 (size=12104) 2024-11-12T06:49:30,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:30,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394230688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:30,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:30,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742260_1436 (size=12001) 2024-11-12T06:49:30,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394230690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:30,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:30,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394230691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:30,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:30,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394230692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:30,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:30,740 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f0beee3d6b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/6bfcb3bfb3734c9a812bab23618e4edf, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/a13016a9637143a5afa4a51adce2fc4b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/3eeb8796b18043309faa71bd9f0942f7] to archive 2024-11-12T06:49:30,741 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f0beee3d6b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T06:49:30,741 WARN [regionserver/9f0beee3d6b6:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-12T06:49:30,742 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f0beee3d6b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/6bfcb3bfb3734c9a812bab23618e4edf to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/6bfcb3bfb3734c9a812bab23618e4edf 2024-11-12T06:49:30,743 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f0beee3d6b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/a13016a9637143a5afa4a51adce2fc4b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/a13016a9637143a5afa4a51adce2fc4b 2024-11-12T06:49:30,744 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f0beee3d6b6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/3eeb8796b18043309faa71bd9f0942f7 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/3eeb8796b18043309faa71bd9f0942f7 2024-11-12T06:49:30,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T06:49:30,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:30,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:30,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:30,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:30,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:30,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:30,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:30,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394230806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:30,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:30,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394230817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:30,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394230817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:30,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394230817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:30,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:30,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:30,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T06:49:30,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:30,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:30,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:30,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:30,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:30,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:31,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394231027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:31,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394231027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394231027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394231027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:31,070 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/0149efffb6f84f8695ba9047e8935bac as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/0149efffb6f84f8695ba9047e8935bac 2024-11-12T06:49:31,074 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efaada0e985d01f31ce286661347290d/B of efaada0e985d01f31ce286661347290d into 0149efffb6f84f8695ba9047e8935bac(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:31,074 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:31,074 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/B, priority=13, startTime=1731394170647; duration=0sec 2024-11-12T06:49:31,074 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:31,074 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:B 2024-11-12T06:49:31,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T06:49:31,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:31,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:31,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:31,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:31,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:31,101 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/59dacdbadb8e42f3be9a07debff7770f as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/59dacdbadb8e42f3be9a07debff7770f 2024-11-12T06:49:31,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/7455e0c16f734e3e8845bea3251689fa 2024-11-12T06:49:31,105 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efaada0e985d01f31ce286661347290d/C of efaada0e985d01f31ce286661347290d into 59dacdbadb8e42f3be9a07debff7770f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:31,105 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:31,105 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/C, priority=13, startTime=1731394170647; duration=0sec 2024-11-12T06:49:31,105 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:31,105 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:C 2024-11-12T06:49:31,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/7c9ef7f61c6f483aaf10e98be81b4316 is 50, key is test_row_0/C:col10/1731394170668/Put/seqid=0 2024-11-12T06:49:31,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742261_1437 (size=12001) 2024-11-12T06:49:31,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T06:49:31,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:31,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:31,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:31,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:31,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:31,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394231337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:31,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:31,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394231347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:31,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:31,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394231347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:31,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:31,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:31,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394231347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:31,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:31,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T06:49:31,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:31,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:31,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:31,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:31,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T06:49:31,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/7c9ef7f61c6f483aaf10e98be81b4316 2024-11-12T06:49:31,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/91e216b1083649a69d81f5cde3d0bccd as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/91e216b1083649a69d81f5cde3d0bccd 2024-11-12T06:49:31,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/91e216b1083649a69d81f5cde3d0bccd, entries=200, sequenceid=94, filesize=14.0 K 2024-11-12T06:49:31,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/7455e0c16f734e3e8845bea3251689fa as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/7455e0c16f734e3e8845bea3251689fa 2024-11-12T06:49:31,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/7455e0c16f734e3e8845bea3251689fa, entries=150, sequenceid=94, filesize=11.7 K 2024-11-12T06:49:31,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/7c9ef7f61c6f483aaf10e98be81b4316 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/7c9ef7f61c6f483aaf10e98be81b4316 2024-11-12T06:49:31,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/7c9ef7f61c6f483aaf10e98be81b4316, entries=150, sequenceid=94, filesize=11.7 K 2024-11-12T06:49:31,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for efaada0e985d01f31ce286661347290d in 862ms, sequenceid=94, compaction requested=false 2024-11-12T06:49:31,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:31,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T06:49:31,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:31,553 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing efaada0e985d01f31ce286661347290d 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-12T06:49:31,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=A 2024-11-12T06:49:31,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:31,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=B 2024-11-12T06:49:31,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:31,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=C 2024-11-12T06:49:31,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:31,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/fda3702b7b8f473fb2fde3b74b511271 is 50, key is test_row_0/A:col10/1731394170691/Put/seqid=0 2024-11-12T06:49:31,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742262_1438 (size=12001) 2024-11-12T06:49:31,564 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/fda3702b7b8f473fb2fde3b74b511271 2024-11-12T06:49:31,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/9e5441dd350140d789238b817de4833f is 50, key is test_row_0/B:col10/1731394170691/Put/seqid=0 2024-11-12T06:49:31,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742263_1439 (size=12001) 2024-11-12T06:49:31,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on efaada0e985d01f31ce286661347290d 2024-11-12T06:49:31,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394171784, max timestamp=1731394171831Num uniques 859; 2024-11-12T06:49:31,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394171784, max timestamp=1731394171831Num uniques 859; 2024-11-12T06:49:31,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394171784, max timestamp=1731394171831Num uniques 859; 2024-11-12T06:49:31,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:31,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:31,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:31,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:31,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394231885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:31,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731394171874, max timestamp=1731394171893Num uniques 1068; 2024-11-12T06:49:31,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731394171874, max timestamp=1731394171893Num uniques 1068; 2024-11-12T06:49:31,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-12T06:49:31,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-12T06:49:31,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731394171874, max timestamp=1731394171893Num uniques 1068; 2024-11-12T06:49:31,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-12T06:49:31,977 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/9e5441dd350140d789238b817de4833f 2024-11-12T06:49:31,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:31,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394231977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:31,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/338710321cd34d98b0ad92bbba717a52 is 50, key is test_row_0/C:col10/1731394170691/Put/seqid=0 2024-11-12T06:49:31,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742264_1440 (size=12001) 2024-11-12T06:49:31,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:31,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394231978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:31,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:31,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:31,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394231979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:31,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:31,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:32,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:32,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394231985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:32,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:32,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:32,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394231997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:32,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:32,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:32,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394232097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:32,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:32,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394232107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:32,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:32,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394232107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:32,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:32,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:32,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:32,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394232107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:32,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:32,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394232207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:32,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:32,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394232317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:32,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:32,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:32,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394232327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:32,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:32,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:32,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394232327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:32,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394232327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:32,396 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/338710321cd34d98b0ad92bbba717a52 2024-11-12T06:49:32,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/fda3702b7b8f473fb2fde3b74b511271 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/fda3702b7b8f473fb2fde3b74b511271 2024-11-12T06:49:32,401 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/fda3702b7b8f473fb2fde3b74b511271, entries=150, sequenceid=105, filesize=11.7 K 2024-11-12T06:49:32,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/9e5441dd350140d789238b817de4833f as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/9e5441dd350140d789238b817de4833f 2024-11-12T06:49:32,405 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/9e5441dd350140d789238b817de4833f, entries=150, sequenceid=105, filesize=11.7 K 2024-11-12T06:49:32,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/338710321cd34d98b0ad92bbba717a52 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/338710321cd34d98b0ad92bbba717a52 2024-11-12T06:49:32,408 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/338710321cd34d98b0ad92bbba717a52, entries=150, sequenceid=105, filesize=11.7 K 2024-11-12T06:49:32,408 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for efaada0e985d01f31ce286661347290d in 855ms, sequenceid=105, compaction requested=true 2024-11-12T06:49:32,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:32,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:32,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-12T06:49:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-12T06:49:32,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-12T06:49:32,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1490 sec 2024-11-12T06:49:32,412 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 3.1540 sec 2024-11-12T06:49:32,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on efaada0e985d01f31ce286661347290d 2024-11-12T06:49:32,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efaada0e985d01f31ce286661347290d 3/3 column families, dataSize=161.02 KB heapSize=450.47 KB 2024-11-12T06:49:32,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=A 2024-11-12T06:49:32,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:49:32,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=B 2024-11-12T06:49:32,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:49:32,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=C 2024-11-12T06:49:32,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:49:32,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/afda0b19426f4a569776bbc573408c5e is 50, key is test_row_0/A:col10/1731394172524/Put/seqid=0 2024-11-12T06:49:32,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742265_1441 (size=14541) 2024-11-12T06:49:32,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:32,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394232562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:32,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:32,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394232627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:32,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394232647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:32,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:32,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394232647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:32,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:32,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:32,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394232647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:32,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:32,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:32,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:32,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394232677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:32,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394232887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:32,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/afda0b19426f4a569776bbc573408c5e 2024-11-12T06:49:32,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/0b957fee8c0e44b09d7c1adb1100f54b is 50, key is test_row_0/B:col10/1731394172524/Put/seqid=0 2024-11-12T06:49:32,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742266_1442 (size=12151) 2024-11-12T06:49:33,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394233147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:33,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:33,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394233157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:33,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:33,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394233167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:33,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:33,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:33,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394233167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:33,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:33,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:33,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:33,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:33,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394233206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:33,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/0b957fee8c0e44b09d7c1adb1100f54b 2024-11-12T06:49:33,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/566a652fa4bb4bcda0a7239771d7b481 is 50, key is test_row_0/C:col10/1731394172524/Put/seqid=0 2024-11-12T06:49:33,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742267_1443 (size=12151) 2024-11-12T06:49:33,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T06:49:33,427 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:49:33,428 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:49:33,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-12T06:49:33,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T06:49:33,429 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:49:33,430 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:49:33,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:49:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T06:49:33,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T06:49:33,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:33,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:33,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:33,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:33,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:33,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:33,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394233717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:33,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:33,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:33,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:33,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T06:49:33,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:33,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:33,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:33,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:33,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:33,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:33,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T06:49:33,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/566a652fa4bb4bcda0a7239771d7b481 2024-11-12T06:49:33,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/afda0b19426f4a569776bbc573408c5e as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/afda0b19426f4a569776bbc573408c5e 2024-11-12T06:49:33,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/afda0b19426f4a569776bbc573408c5e, entries=200, sequenceid=160, filesize=14.2 K 2024-11-12T06:49:33,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/0b957fee8c0e44b09d7c1adb1100f54b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/0b957fee8c0e44b09d7c1adb1100f54b 2024-11-12T06:49:33,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/0b957fee8c0e44b09d7c1adb1100f54b, entries=150, sequenceid=160, filesize=11.9 K 2024-11-12T06:49:33,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/566a652fa4bb4bcda0a7239771d7b481 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/566a652fa4bb4bcda0a7239771d7b481 2024-11-12T06:49:33,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/566a652fa4bb4bcda0a7239771d7b481, entries=150, sequenceid=160, filesize=11.9 K 2024-11-12T06:49:33,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~450.42 KB/461232, currentSize=26.84 KB/27480 for efaada0e985d01f31ce286661347290d in 1248ms, sequenceid=160, compaction requested=true 2024-11-12T06:49:33,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:33,774 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:49:33,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:49:33,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:33,774 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:49:33,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:49:33,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:33,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:49:33,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:33,775 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52987 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:49:33,775 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/A is initiating minor compaction (all files) 2024-11-12T06:49:33,775 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/A in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:33,775 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/0f7a8b4b4902472b902c30638a711d6c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/91e216b1083649a69d81f5cde3d0bccd, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/fda3702b7b8f473fb2fde3b74b511271, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/afda0b19426f4a569776bbc573408c5e] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=51.7 K 2024-11-12T06:49:33,775 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:49:33,775 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f7a8b4b4902472b902c30638a711d6c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731394169366 2024-11-12T06:49:33,775 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/B is initiating minor compaction (all files) 2024-11-12T06:49:33,775 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/B in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:33,775 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/0149efffb6f84f8695ba9047e8935bac, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/7455e0c16f734e3e8845bea3251689fa, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/9e5441dd350140d789238b817de4833f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/0b957fee8c0e44b09d7c1adb1100f54b] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=47.1 K 2024-11-12T06:49:33,776 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91e216b1083649a69d81f5cde3d0bccd, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731394169459 2024-11-12T06:49:33,776 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 0149efffb6f84f8695ba9047e8935bac, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731394169366 2024-11-12T06:49:33,776 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting fda3702b7b8f473fb2fde3b74b511271, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731394170686 2024-11-12T06:49:33,776 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 7455e0c16f734e3e8845bea3251689fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731394169459 2024-11-12T06:49:33,776 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting afda0b19426f4a569776bbc573408c5e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731394171975 2024-11-12T06:49:33,776 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e5441dd350140d789238b817de4833f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731394170686 2024-11-12T06:49:33,777 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b957fee8c0e44b09d7c1adb1100f54b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731394171976 2024-11-12T06:49:33,788 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#B#compaction#366 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:33,788 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/b84529bdee2c445785e856098731cd10 is 50, key is test_row_0/B:col10/1731394172524/Put/seqid=0 2024-11-12T06:49:33,789 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#A#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:33,790 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/2a6db3a05f6c405e8eee3fcf1eb6cc9c is 50, key is test_row_0/A:col10/1731394172524/Put/seqid=0 2024-11-12T06:49:33,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742268_1444 (size=12391) 2024-11-12T06:49:33,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742269_1445 (size=12289) 2024-11-12T06:49:33,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T06:49:33,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:33,887 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing efaada0e985d01f31ce286661347290d 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-12T06:49:33,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=A 2024-11-12T06:49:33,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:33,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=B 2024-11-12T06:49:33,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:33,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=C 2024-11-12T06:49:33,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:33,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/69a229600e314b168e7359cc33dacd3f is 50, key is test_row_1/A:col10/1731394172560/Put/seqid=0 2024-11-12T06:49:33,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742270_1446 (size=9757) 2024-11-12T06:49:34,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T06:49:34,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on efaada0e985d01f31ce286661347290d 2024-11-12T06:49:34,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:34,200 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/b84529bdee2c445785e856098731cd10 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/b84529bdee2c445785e856098731cd10 2024-11-12T06:49:34,201 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/2a6db3a05f6c405e8eee3fcf1eb6cc9c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/2a6db3a05f6c405e8eee3fcf1eb6cc9c 2024-11-12T06:49:34,203 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efaada0e985d01f31ce286661347290d/B of efaada0e985d01f31ce286661347290d into b84529bdee2c445785e856098731cd10(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:34,203 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:34,203 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/B, priority=12, startTime=1731394173774; duration=0sec 2024-11-12T06:49:34,203 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:34,203 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:B 2024-11-12T06:49:34,203 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efaada0e985d01f31ce286661347290d/A of efaada0e985d01f31ce286661347290d into 2a6db3a05f6c405e8eee3fcf1eb6cc9c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:34,203 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:49:34,203 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:34,203 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/A, priority=12, startTime=1731394173774; duration=0sec 2024-11-12T06:49:34,203 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:34,203 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:A 2024-11-12T06:49:34,204 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:49:34,204 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/C is initiating minor compaction (all files) 2024-11-12T06:49:34,204 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/C in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:34,204 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/59dacdbadb8e42f3be9a07debff7770f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/7c9ef7f61c6f483aaf10e98be81b4316, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/338710321cd34d98b0ad92bbba717a52, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/566a652fa4bb4bcda0a7239771d7b481] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=47.1 K 2024-11-12T06:49:34,204 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 59dacdbadb8e42f3be9a07debff7770f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731394169366 2024-11-12T06:49:34,205 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c9ef7f61c6f483aaf10e98be81b4316, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731394169459 2024-11-12T06:49:34,205 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 338710321cd34d98b0ad92bbba717a52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731394170686 2024-11-12T06:49:34,205 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 566a652fa4bb4bcda0a7239771d7b481, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731394171976 2024-11-12T06:49:34,213 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#C#compaction#369 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:34,213 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/c331efbd97854a62b5cd7563a95430de is 50, key is test_row_0/C:col10/1731394172524/Put/seqid=0 2024-11-12T06:49:34,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731394174194, max timestamp=1731394174211Num uniques 859; 2024-11-12T06:49:34,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731394174194, max timestamp=1731394174211Num uniques 859; 2024-11-12T06:49:34,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T06:49:34,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T06:49:34,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731394174194, max timestamp=1731394174211Num uniques 859; 2024-11-12T06:49:34,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T06:49:34,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742271_1447 (size=12391) 2024-11-12T06:49:34,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:34,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394234248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:34,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:34,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394234252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:34,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:34,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394234253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:34,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731394174241, max timestamp=1731394174256Num uniques 1168; 2024-11-12T06:49:34,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=12268, suffix heap size=149440,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=268 2024-11-12T06:49:34,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:34,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731394174241, max timestamp=1731394174256Num uniques 1168; 2024-11-12T06:49:34,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=12268, suffix heap size=149440,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=268 2024-11-12T06:49:34,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731394174241, max timestamp=1731394174256Num uniques 1168; 2024-11-12T06:49:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=12268, suffix heap size=149440,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=268 2024-11-12T06:49:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:34,294 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/69a229600e314b168e7359cc33dacd3f 2024-11-12T06:49:34,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/26eb542f04f249f899c0b272fddbbcb9 is 50, key is test_row_1/B:col10/1731394172560/Put/seqid=0 2024-11-12T06:49:34,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742272_1448 (size=9757) 2024-11-12T06:49:34,303 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/26eb542f04f249f899c0b272fddbbcb9 2024-11-12T06:49:34,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/88729fb5bc6c4ce88ae2984d005051fa is 50, key is test_row_1/C:col10/1731394172560/Put/seqid=0 2024-11-12T06:49:34,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742273_1449 (size=9757) 2024-11-12T06:49:34,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:34,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394234367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:34,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:34,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394234368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:34,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:34,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731394174332, max timestamp=1731394174379Num uniques 1127; 2024-11-12T06:49:34,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-12T06:49:34,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:34,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731394174332, max timestamp=1731394174379Num uniques 1127; 2024-11-12T06:49:34,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731394174332, max timestamp=1731394174379Num uniques 1127; 2024-11-12T06:49:34,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-12T06:49:34,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-12T06:49:34,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731394174428, max timestamp=1731394174453Num uniques 1086; 2024-11-12T06:49:34,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-12T06:49:34,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731394174428, max timestamp=1731394174453Num uniques 1086; 2024-11-12T06:49:34,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-12T06:49:34,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731394174428, max timestamp=1731394174453Num uniques 1086; 2024-11-12T06:49:34,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-12T06:49:34,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731394174527, max timestamp=1731394174558Num uniques 1095; 2024-11-12T06:49:34,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-12T06:49:34,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731394174527, max timestamp=1731394174558Num uniques 1095; 2024-11-12T06:49:34,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-12T06:49:34,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731394174527, max timestamp=1731394174558Num uniques 1095; 2024-11-12T06:49:34,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-12T06:49:34,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T06:49:34,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:34,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394234611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:34,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731394174607, max timestamp=1731394174621Num uniques 1104; 2024-11-12T06:49:34,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731394174607, max timestamp=1731394174621Num uniques 1104; 2024-11-12T06:49:34,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-12T06:49:34,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-12T06:49:34,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731394174607, max timestamp=1731394174621Num uniques 1104; 2024-11-12T06:49:34,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-12T06:49:34,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:34,627 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/c331efbd97854a62b5cd7563a95430de as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/c331efbd97854a62b5cd7563a95430de 2024-11-12T06:49:34,633 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efaada0e985d01f31ce286661347290d/C of efaada0e985d01f31ce286661347290d into c331efbd97854a62b5cd7563a95430de(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:34,633 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:34,633 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/C, priority=12, startTime=1731394173774; duration=0sec 2024-11-12T06:49:34,633 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:34,633 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:C 2024-11-12T06:49:34,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1731394174660, max timestamp=1731394174679Num uniques 1113; 2024-11-12T06:49:34,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1731394174660, max timestamp=1731394174679Num uniques 1113; 2024-11-12T06:49:34,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-11-12T06:49:34,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-11-12T06:49:34,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1731394174660, max timestamp=1731394174679Num uniques 1113; 2024-11-12T06:49:34,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-11-12T06:49:34,718 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/88729fb5bc6c4ce88ae2984d005051fa 2024-11-12T06:49:34,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/69a229600e314b168e7359cc33dacd3f as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/69a229600e314b168e7359cc33dacd3f 2024-11-12T06:49:34,726 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/69a229600e314b168e7359cc33dacd3f, entries=100, sequenceid=167, filesize=9.5 K 2024-11-12T06:49:34,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/26eb542f04f249f899c0b272fddbbcb9 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/26eb542f04f249f899c0b272fddbbcb9 2024-11-12T06:49:34,729 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/26eb542f04f249f899c0b272fddbbcb9, entries=100, sequenceid=167, filesize=9.5 K 2024-11-12T06:49:34,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/88729fb5bc6c4ce88ae2984d005051fa as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/88729fb5bc6c4ce88ae2984d005051fa 2024-11-12T06:49:34,732 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/88729fb5bc6c4ce88ae2984d005051fa, entries=100, sequenceid=167, filesize=9.5 K 2024-11-12T06:49:34,733 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=134.18 KB/137400 for efaada0e985d01f31ce286661347290d in 846ms, sequenceid=167, compaction requested=false 2024-11-12T06:49:34,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:34,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:34,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-12T06:49:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-12T06:49:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on efaada0e985d01f31ce286661347290d 2024-11-12T06:49:34,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efaada0e985d01f31ce286661347290d 3/3 column families, dataSize=147.60 KB heapSize=415.08 KB 2024-11-12T06:49:34,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=A 2024-11-12T06:49:34,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:49:34,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=B 2024-11-12T06:49:34,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:49:34,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=C 2024-11-12T06:49:34,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:49:34,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-12T06:49:34,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3070 sec 2024-11-12T06:49:34,744 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.3150 sec 2024-11-12T06:49:34,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/73d1460d4bda4f5cac4088e229ebd762 is 50, key is test_row_0/A:col10/1731394174736/Put/seqid=0 2024-11-12T06:49:34,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742274_1450 (size=17181) 2024-11-12T06:49:34,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394234743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:34,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:34,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394234744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:34,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394234745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:34,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394234746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:34,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394234750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:34,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:34,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:34,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394234857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:34,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394234867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:34,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:34,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394234867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:34,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:34,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394234868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:34,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:34,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:34,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394234868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:35,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394235077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:35,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:35,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394235087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394235087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394235088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394235088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:35,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/73d1460d4bda4f5cac4088e229ebd762 2024-11-12T06:49:35,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/19e071138ef44d1b812623f74276165a is 50, key is test_row_0/B:col10/1731394174736/Put/seqid=0 2024-11-12T06:49:35,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742275_1451 (size=12301) 2024-11-12T06:49:35,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394235397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:35,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394235407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394235407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394235407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394235408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:35,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/19e071138ef44d1b812623f74276165a 2024-11-12T06:49:35,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/cf2f1a0a53de48299bd099ebb9250bf8 is 50, key is test_row_0/C:col10/1731394174736/Put/seqid=0 2024-11-12T06:49:35,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742276_1452 (size=12301) 2024-11-12T06:49:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T06:49:35,577 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:49:35,577 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:49:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-12T06:49:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T06:49:35,579 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:49:35,579 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:49:35,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:49:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T06:49:35,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T06:49:35,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:35,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:35,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:35,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:35,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:35,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T06:49:35,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:35,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:35,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:35,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:35,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T06:49:35,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394235917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:35,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:35,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394235927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:35,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:35,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394235927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:35,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:35,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:35,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394235927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:35,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:35,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394235927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:35,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:35,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:35,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/cf2f1a0a53de48299bd099ebb9250bf8 2024-11-12T06:49:35,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/73d1460d4bda4f5cac4088e229ebd762 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/73d1460d4bda4f5cac4088e229ebd762 2024-11-12T06:49:35,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/73d1460d4bda4f5cac4088e229ebd762, entries=250, sequenceid=311, filesize=16.8 K 2024-11-12T06:49:35,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/19e071138ef44d1b812623f74276165a as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/19e071138ef44d1b812623f74276165a 2024-11-12T06:49:35,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/19e071138ef44d1b812623f74276165a, entries=150, sequenceid=311, filesize=12.0 K 2024-11-12T06:49:35,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/cf2f1a0a53de48299bd099ebb9250bf8 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/cf2f1a0a53de48299bd099ebb9250bf8 2024-11-12T06:49:35,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/cf2f1a0a53de48299bd099ebb9250bf8, entries=150, sequenceid=311, filesize=12.0 K 2024-11-12T06:49:35,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~435.66 KB/446112, currentSize=26.84 KB/27480 for efaada0e985d01f31ce286661347290d in 1247ms, sequenceid=311, compaction requested=true 2024-11-12T06:49:35,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:35,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:49:35,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:35,984 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:35,984 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:35,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:49:35,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:35,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:49:35,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:35,985 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39227 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:35,985 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:35,985 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/A is initiating minor compaction (all files) 2024-11-12T06:49:35,985 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/B is initiating minor compaction (all files) 2024-11-12T06:49:35,985 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/A in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:35,986 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/B in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:35,986 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/2a6db3a05f6c405e8eee3fcf1eb6cc9c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/69a229600e314b168e7359cc33dacd3f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/73d1460d4bda4f5cac4088e229ebd762] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=38.3 K 2024-11-12T06:49:35,986 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/b84529bdee2c445785e856098731cd10, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/26eb542f04f249f899c0b272fddbbcb9, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/19e071138ef44d1b812623f74276165a] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=33.6 K 2024-11-12T06:49:35,986 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a6db3a05f6c405e8eee3fcf1eb6cc9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731394171976 2024-11-12T06:49:35,986 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting b84529bdee2c445785e856098731cd10, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731394171976 2024-11-12T06:49:35,986 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 26eb542f04f249f899c0b272fddbbcb9, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731394172546 2024-11-12T06:49:35,986 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69a229600e314b168e7359cc33dacd3f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731394172546 2024-11-12T06:49:35,986 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73d1460d4bda4f5cac4088e229ebd762, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731394174721 2024-11-12T06:49:35,986 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 19e071138ef44d1b812623f74276165a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731394174734 2024-11-12T06:49:35,991 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#B#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:35,992 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/ebdfdfc6e76e466295750d58d0535375 is 50, key is test_row_0/B:col10/1731394174736/Put/seqid=0 2024-11-12T06:49:35,994 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#A#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:35,994 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/5af56074f6db4333a9a5e0e9df351c87 is 50, key is test_row_0/A:col10/1731394174736/Put/seqid=0 2024-11-12T06:49:35,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742277_1453 (size=12643) 2024-11-12T06:49:36,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742278_1454 (size=12541) 2024-11-12T06:49:36,006 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/ebdfdfc6e76e466295750d58d0535375 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/ebdfdfc6e76e466295750d58d0535375 2024-11-12T06:49:36,008 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/5af56074f6db4333a9a5e0e9df351c87 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/5af56074f6db4333a9a5e0e9df351c87 2024-11-12T06:49:36,010 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efaada0e985d01f31ce286661347290d/B of efaada0e985d01f31ce286661347290d into ebdfdfc6e76e466295750d58d0535375(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:36,010 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:36,010 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/B, priority=13, startTime=1731394175984; duration=0sec 2024-11-12T06:49:36,010 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:36,010 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:B 2024-11-12T06:49:36,011 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:36,011 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efaada0e985d01f31ce286661347290d/A of efaada0e985d01f31ce286661347290d into 5af56074f6db4333a9a5e0e9df351c87(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:36,011 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:36,011 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/A, priority=13, startTime=1731394175984; duration=0sec 2024-11-12T06:49:36,011 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:36,011 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:36,011 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:A 2024-11-12T06:49:36,011 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/C is initiating minor compaction (all files) 2024-11-12T06:49:36,011 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/C in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:36,011 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/c331efbd97854a62b5cd7563a95430de, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/88729fb5bc6c4ce88ae2984d005051fa, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/cf2f1a0a53de48299bd099ebb9250bf8] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=33.6 K 2024-11-12T06:49:36,012 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting c331efbd97854a62b5cd7563a95430de, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731394171976 2024-11-12T06:49:36,012 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 88729fb5bc6c4ce88ae2984d005051fa, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731394172546 2024-11-12T06:49:36,012 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting cf2f1a0a53de48299bd099ebb9250bf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731394174734 2024-11-12T06:49:36,018 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#C#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:36,019 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/6d05d49a246b44428c1c824b7ad20608 is 50, key is test_row_0/C:col10/1731394174736/Put/seqid=0 2024-11-12T06:49:36,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742279_1455 (size=12643) 2024-11-12T06:49:36,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T06:49:36,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:36,037 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing efaada0e985d01f31ce286661347290d 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-12T06:49:36,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=A 2024-11-12T06:49:36,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:36,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=B 2024-11-12T06:49:36,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:36,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=C 2024-11-12T06:49:36,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:36,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/ad2c55db68224deea8722255c6df169b is 50, key is test_row_0/A:col10/1731394174745/Put/seqid=0 2024-11-12T06:49:36,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742280_1456 (size=12301) 2024-11-12T06:49:36,056 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/ad2c55db68224deea8722255c6df169b 2024-11-12T06:49:36,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/5b91adae434745749b0c7f058635fbfd is 50, key is test_row_0/B:col10/1731394174745/Put/seqid=0 2024-11-12T06:49:36,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742281_1457 (size=12301) 2024-11-12T06:49:36,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T06:49:36,428 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/6d05d49a246b44428c1c824b7ad20608 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/6d05d49a246b44428c1c824b7ad20608 2024-11-12T06:49:36,432 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efaada0e985d01f31ce286661347290d/C of efaada0e985d01f31ce286661347290d into 6d05d49a246b44428c1c824b7ad20608(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:36,432 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:36,432 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/C, priority=13, startTime=1731394175985; duration=0sec 2024-11-12T06:49:36,432 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:36,432 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:C 2024-11-12T06:49:36,466 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/5b91adae434745749b0c7f058635fbfd 2024-11-12T06:49:36,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/c50f7212b5754fccafb141414cc86362 is 50, key is test_row_0/C:col10/1731394174745/Put/seqid=0 2024-11-12T06:49:36,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742282_1458 (size=12301) 2024-11-12T06:49:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T06:49:36,888 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/c50f7212b5754fccafb141414cc86362 2024-11-12T06:49:36,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/ad2c55db68224deea8722255c6df169b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/ad2c55db68224deea8722255c6df169b 2024-11-12T06:49:36,894 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/ad2c55db68224deea8722255c6df169b, entries=150, sequenceid=320, filesize=12.0 K 2024-11-12T06:49:36,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/5b91adae434745749b0c7f058635fbfd as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/5b91adae434745749b0c7f058635fbfd 2024-11-12T06:49:36,900 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/5b91adae434745749b0c7f058635fbfd, entries=150, sequenceid=320, filesize=12.0 K 2024-11-12T06:49:36,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/c50f7212b5754fccafb141414cc86362 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/c50f7212b5754fccafb141414cc86362 2024-11-12T06:49:36,903 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/c50f7212b5754fccafb141414cc86362, entries=150, sequenceid=320, filesize=12.0 K 2024-11-12T06:49:36,904 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for efaada0e985d01f31ce286661347290d in 867ms, sequenceid=320, compaction requested=false 2024-11-12T06:49:36,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:36,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:36,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-12T06:49:36,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-12T06:49:36,906 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-12T06:49:36,906 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3250 sec 2024-11-12T06:49:36,908 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.3300 sec 2024-11-12T06:49:36,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on efaada0e985d01f31ce286661347290d 2024-11-12T06:49:36,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efaada0e985d01f31ce286661347290d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T06:49:36,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=A 2024-11-12T06:49:36,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:36,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=B 2024-11-12T06:49:36,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:36,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=C 2024-11-12T06:49:36,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:36,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/385770d83bf949c48cd9a9373635a135 is 50, key is test_row_0/A:col10/1731394176931/Put/seqid=0 2024-11-12T06:49:36,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742283_1459 (size=17181) 2024-11-12T06:49:36,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394176968, max timestamp=1731394176979Num uniques 859; 2024-11-12T06:49:36,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:49:36,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394176968, max timestamp=1731394176979Num uniques 859; 2024-11-12T06:49:36,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394176968, max timestamp=1731394176979Num uniques 859; 2024-11-12T06:49:36,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:49:36,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:49:37,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:37,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394237012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:37,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:37,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:37,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:37,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394237012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:37,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394237012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:37,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:37,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394237013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:37,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394237014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:37,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394237117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:37,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:37,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:37,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394237127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:37,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394237137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394237137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:37,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394237137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:37,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394237327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:37,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/385770d83bf949c48cd9a9373635a135 2024-11-12T06:49:37,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394237347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:37,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:37,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/e3b45b147b124011b7ae509eb50014bb is 50, key is test_row_0/B:col10/1731394176931/Put/seqid=0 2024-11-12T06:49:37,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742284_1460 (size=12301) 2024-11-12T06:49:37,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394237357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:37,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:37,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394237357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:37,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:37,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394237357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:37,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:37,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:37,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:37,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394237636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:37,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:37,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:37,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394237657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:37,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:37,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394237677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:37,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:37,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394237677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:37,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394237677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:37,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T06:49:37,727 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:49:37,727 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:49:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-12T06:49:37,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T06:49:37,729 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:49:37,729 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:49:37,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:49:37,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/e3b45b147b124011b7ae509eb50014bb 2024-11-12T06:49:37,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/f3634668dfd34c2aa8793f865d18d783 is 50, key is test_row_0/C:col10/1731394176931/Put/seqid=0 2024-11-12T06:49:37,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742285_1461 (size=12301) 2024-11-12T06:49:37,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/f3634668dfd34c2aa8793f865d18d783 2024-11-12T06:49:37,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/385770d83bf949c48cd9a9373635a135 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/385770d83bf949c48cd9a9373635a135 2024-11-12T06:49:37,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/385770d83bf949c48cd9a9373635a135, entries=250, sequenceid=331, filesize=16.8 K 2024-11-12T06:49:37,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/e3b45b147b124011b7ae509eb50014bb as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/e3b45b147b124011b7ae509eb50014bb 2024-11-12T06:49:37,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/e3b45b147b124011b7ae509eb50014bb, entries=150, sequenceid=331, filesize=12.0 K 2024-11-12T06:49:37,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/f3634668dfd34c2aa8793f865d18d783 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/f3634668dfd34c2aa8793f865d18d783 2024-11-12T06:49:37,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/f3634668dfd34c2aa8793f865d18d783, entries=150, sequenceid=331, filesize=12.0 K 2024-11-12T06:49:37,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for efaada0e985d01f31ce286661347290d in 842ms, sequenceid=331, compaction requested=true 2024-11-12T06:49:37,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:37,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:49:37,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:37,784 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:37,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:49:37,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:37,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:49:37,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T06:49:37,784 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:37,785 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:37,785 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:37,785 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/B is initiating minor compaction (all files) 2024-11-12T06:49:37,785 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/A is initiating minor compaction (all files) 2024-11-12T06:49:37,785 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/B in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:37,785 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/A in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:37,785 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/ebdfdfc6e76e466295750d58d0535375, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/5b91adae434745749b0c7f058635fbfd, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/e3b45b147b124011b7ae509eb50014bb] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=36.4 K 2024-11-12T06:49:37,785 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/5af56074f6db4333a9a5e0e9df351c87, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/ad2c55db68224deea8722255c6df169b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/385770d83bf949c48cd9a9373635a135] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=41.0 K 2024-11-12T06:49:37,785 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5af56074f6db4333a9a5e0e9df351c87, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731394174734 2024-11-12T06:49:37,785 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting ebdfdfc6e76e466295750d58d0535375, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731394174734 2024-11-12T06:49:37,785 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad2c55db68224deea8722255c6df169b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731394174741 2024-11-12T06:49:37,785 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b91adae434745749b0c7f058635fbfd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731394174741 2024-11-12T06:49:37,785 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 385770d83bf949c48cd9a9373635a135, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731394176928 2024-11-12T06:49:37,785 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting e3b45b147b124011b7ae509eb50014bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731394176931 2024-11-12T06:49:37,792 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#B#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:37,792 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/fc32667035f34480bd5c39e62f2aa227 is 50, key is test_row_0/B:col10/1731394176931/Put/seqid=0 2024-11-12T06:49:37,795 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#A#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:37,796 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/b9c982a7d37446cc994aa11096a0e994 is 50, key is test_row_0/A:col10/1731394176931/Put/seqid=0 2024-11-12T06:49:37,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742286_1462 (size=12745) 2024-11-12T06:49:37,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742287_1463 (size=12643) 2024-11-12T06:49:37,812 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/fc32667035f34480bd5c39e62f2aa227 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/fc32667035f34480bd5c39e62f2aa227 2024-11-12T06:49:37,817 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efaada0e985d01f31ce286661347290d/B of efaada0e985d01f31ce286661347290d into fc32667035f34480bd5c39e62f2aa227(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:37,817 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:37,818 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/B, priority=13, startTime=1731394177784; duration=0sec 2024-11-12T06:49:37,818 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:37,818 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:B 2024-11-12T06:49:37,818 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:37,819 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:37,819 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/C is initiating minor compaction (all files) 2024-11-12T06:49:37,819 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/C in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:37,819 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/6d05d49a246b44428c1c824b7ad20608, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/c50f7212b5754fccafb141414cc86362, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/f3634668dfd34c2aa8793f865d18d783] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=36.4 K 2024-11-12T06:49:37,819 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d05d49a246b44428c1c824b7ad20608, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731394174734 2024-11-12T06:49:37,819 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting c50f7212b5754fccafb141414cc86362, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731394174741 2024-11-12T06:49:37,820 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting f3634668dfd34c2aa8793f865d18d783, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731394176931 2024-11-12T06:49:37,827 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#C#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:37,827 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/e547c2d018444e1cbcae5a28e4b9c5e3 is 50, key is test_row_0/C:col10/1731394176931/Put/seqid=0 2024-11-12T06:49:37,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742288_1464 (size=12745) 2024-11-12T06:49:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T06:49:37,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T06:49:37,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:37,881 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing efaada0e985d01f31ce286661347290d 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-12T06:49:37,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=A 2024-11-12T06:49:37,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:37,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=B 2024-11-12T06:49:37,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:37,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=C 2024-11-12T06:49:37,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:37,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/8863092ea68144cda37116a944949d3c is 50, key is test_row_0/A:col10/1731394177011/Put/seqid=0 2024-11-12T06:49:37,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742289_1465 (size=12301) 2024-11-12T06:49:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T06:49:38,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on efaada0e985d01f31ce286661347290d 2024-11-12T06:49:38,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:38,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:38,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394238184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:38,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:38,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394238186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:38,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:38,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394238187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:38,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:38,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394238190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:38,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:38,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394238197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:38,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:38,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:38,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:38,211 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/b9c982a7d37446cc994aa11096a0e994 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/b9c982a7d37446cc994aa11096a0e994 2024-11-12T06:49:38,214 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efaada0e985d01f31ce286661347290d/A of efaada0e985d01f31ce286661347290d into b9c982a7d37446cc994aa11096a0e994(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:38,214 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:38,214 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/A, priority=13, startTime=1731394177784; duration=0sec 2024-11-12T06:49:38,215 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:38,215 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:A 2024-11-12T06:49:38,234 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/e547c2d018444e1cbcae5a28e4b9c5e3 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/e547c2d018444e1cbcae5a28e4b9c5e3 2024-11-12T06:49:38,238 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efaada0e985d01f31ce286661347290d/C of efaada0e985d01f31ce286661347290d into e547c2d018444e1cbcae5a28e4b9c5e3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:38,238 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:38,238 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/C, priority=13, startTime=1731394177784; duration=0sec 2024-11-12T06:49:38,238 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:38,238 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:C 2024-11-12T06:49:38,293 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/8863092ea68144cda37116a944949d3c 2024-11-12T06:49:38,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/22436698610f41f1b13f94ffee766446 is 50, key is test_row_0/B:col10/1731394177011/Put/seqid=0 2024-11-12T06:49:38,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742290_1466 (size=12301) 2024-11-12T06:49:38,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394238297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:38,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394238307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:38,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:38,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:38,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:38,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T06:49:38,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394238517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:38,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:38,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:38,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:38,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394238527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:38,702 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/22436698610f41f1b13f94ffee766446 2024-11-12T06:49:38,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/9a6392785bc648fa89f8269601fcb7a5 is 50, key is test_row_0/C:col10/1731394177011/Put/seqid=0 2024-11-12T06:49:38,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742291_1467 (size=12301) 2024-11-12T06:49:38,715 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/9a6392785bc648fa89f8269601fcb7a5 2024-11-12T06:49:38,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/8863092ea68144cda37116a944949d3c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/8863092ea68144cda37116a944949d3c 2024-11-12T06:49:38,721 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/8863092ea68144cda37116a944949d3c, entries=150, sequenceid=369, filesize=12.0 K 2024-11-12T06:49:38,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/22436698610f41f1b13f94ffee766446 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/22436698610f41f1b13f94ffee766446 2024-11-12T06:49:38,725 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/22436698610f41f1b13f94ffee766446, entries=150, sequenceid=369, filesize=12.0 K 2024-11-12T06:49:38,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/9a6392785bc648fa89f8269601fcb7a5 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/9a6392785bc648fa89f8269601fcb7a5 2024-11-12T06:49:38,729 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/9a6392785bc648fa89f8269601fcb7a5, entries=150, sequenceid=369, filesize=12.0 K 2024-11-12T06:49:38,730 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for efaada0e985d01f31ce286661347290d in 849ms, sequenceid=369, compaction requested=false 2024-11-12T06:49:38,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:38,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:38,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-12T06:49:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-12T06:49:38,732 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-12T06:49:38,732 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0020 sec 2024-11-12T06:49:38,734 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.0060 sec 2024-11-12T06:49:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on efaada0e985d01f31ce286661347290d 2024-11-12T06:49:38,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efaada0e985d01f31ce286661347290d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T06:49:38,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=A 2024-11-12T06:49:38,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:38,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=B 2024-11-12T06:49:38,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:38,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=C 2024-11-12T06:49:38,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:38,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/5a26ee24add04aa09a0cea692e27b203 is 50, key is test_row_0/A:col10/1731394178842/Put/seqid=0 2024-11-12T06:49:38,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742292_1468 (size=14741) 2024-11-12T06:49:38,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T06:49:38,867 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:49:38,868 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:49:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-12T06:49:38,869 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:49:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T06:49:38,870 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:49:38,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:49:38,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394178903, max timestamp=1731394178920Num uniques 859; 2024-11-12T06:49:38,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:38,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394178903, max timestamp=1731394178920Num uniques 859; 2024-11-12T06:49:38,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:38,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394178903, max timestamp=1731394178920Num uniques 859; 2024-11-12T06:49:38,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T06:49:38,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:38,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394238983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:38,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:38,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:38,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:38,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394238987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:39,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T06:49:39,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:39,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:39,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:39,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:39,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:39,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:39,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394239097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:39,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:39,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:39,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:39,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:39,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394239107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:39,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:39,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:39,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:39,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T06:49:39,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:39,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:39,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:39,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:39,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T06:49:39,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394239207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:39,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394239207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:39,215 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_2' in efaada0e985d01f31ce286661347290d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:49:39,215 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_1' in efaada0e985d01f31ce286661347290d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:49:39,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:39,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:39,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:39,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:39,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:39,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:39,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394239216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:39,228 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_1' in efaada0e985d01f31ce286661347290d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:49:39,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:39,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:39,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:39,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/5a26ee24add04aa09a0cea692e27b203 2024-11-12T06:49:39,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/9750181af3c54b35973d5852cebfef0c is 50, key is test_row_0/B:col10/1731394178842/Put/seqid=0 2024-11-12T06:49:39,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742293_1469 (size=12301) 2024-11-12T06:49:39,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394239307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:39,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T06:49:39,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:39,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:39,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:39,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:39,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:39,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394239326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:39,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:39,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:39,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:39,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T06:49:39,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:39,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:39,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:39,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:39,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T06:49:39,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T06:49:39,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:39,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:39,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:39,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:39,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:39,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394239627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:39,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:39,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:39,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:39,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394239647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:39,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:39,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:39,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:39,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/9750181af3c54b35973d5852cebfef0c 2024-11-12T06:49:39,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/22de97d685bd4bc9af6db03e93b36f35 is 50, key is test_row_0/C:col10/1731394178842/Put/seqid=0 2024-11-12T06:49:39,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742294_1470 (size=12301) 2024-11-12T06:49:39,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T06:49:39,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:39,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:39,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:39,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:39,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:39,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T06:49:39,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:39,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:39,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:39,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:39,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T06:49:40,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/22de97d685bd4bc9af6db03e93b36f35 2024-11-12T06:49:40,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/5a26ee24add04aa09a0cea692e27b203 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/5a26ee24add04aa09a0cea692e27b203 2024-11-12T06:49:40,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/5a26ee24add04aa09a0cea692e27b203, entries=200, sequenceid=382, filesize=14.4 K 2024-11-12T06:49:40,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/9750181af3c54b35973d5852cebfef0c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/9750181af3c54b35973d5852cebfef0c 2024-11-12T06:49:40,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/9750181af3c54b35973d5852cebfef0c, entries=150, sequenceid=382, filesize=12.0 K 2024-11-12T06:49:40,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/22de97d685bd4bc9af6db03e93b36f35 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/22de97d685bd4bc9af6db03e93b36f35 2024-11-12T06:49:40,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/22de97d685bd4bc9af6db03e93b36f35, entries=150, sequenceid=382, filesize=12.0 K 2024-11-12T06:49:40,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for efaada0e985d01f31ce286661347290d in 1237ms, sequenceid=382, compaction requested=true 2024-11-12T06:49:40,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:40,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:49:40,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:40,080 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:40,080 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:40,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:49:40,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:40,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:49:40,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:40,081 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:40,081 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:40,081 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/A is initiating minor compaction (all files) 2024-11-12T06:49:40,081 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/B is initiating minor compaction (all files) 2024-11-12T06:49:40,081 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/B in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:40,081 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/A in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:40,081 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/b9c982a7d37446cc994aa11096a0e994, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/8863092ea68144cda37116a944949d3c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/5a26ee24add04aa09a0cea692e27b203] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=38.8 K 2024-11-12T06:49:40,081 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/fc32667035f34480bd5c39e62f2aa227, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/22436698610f41f1b13f94ffee766446, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/9750181af3c54b35973d5852cebfef0c] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=36.5 K 2024-11-12T06:49:40,081 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9c982a7d37446cc994aa11096a0e994, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731394176931 2024-11-12T06:49:40,081 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting fc32667035f34480bd5c39e62f2aa227, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731394176931 2024-11-12T06:49:40,082 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 22436698610f41f1b13f94ffee766446, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731394177011 2024-11-12T06:49:40,082 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8863092ea68144cda37116a944949d3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731394177011 2024-11-12T06:49:40,082 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 9750181af3c54b35973d5852cebfef0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731394178183 2024-11-12T06:49:40,082 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a26ee24add04aa09a0cea692e27b203, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731394178179 2024-11-12T06:49:40,088 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#B#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:40,089 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/a575a076060a4172937946fc5d91f201 is 50, key is test_row_0/B:col10/1731394178842/Put/seqid=0 2024-11-12T06:49:40,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T06:49:40,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:40,090 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing efaada0e985d01f31ce286661347290d 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-12T06:49:40,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=A 2024-11-12T06:49:40,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:40,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=B 2024-11-12T06:49:40,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:40,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=C 2024-11-12T06:49:40,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:40,092 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#A#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:40,092 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/9b859f0941f942da8fedbf4d4935f585 is 50, key is test_row_0/A:col10/1731394178842/Put/seqid=0 2024-11-12T06:49:40,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742295_1471 (size=12847) 2024-11-12T06:49:40,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/ea0186fecdcc4324a612c6464531cda7 is 50, key is test_row_0/A:col10/1731394178959/Put/seqid=0 2024-11-12T06:49:40,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742296_1472 (size=12745) 2024-11-12T06:49:40,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742297_1473 (size=12301) 2024-11-12T06:49:40,117 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/ea0186fecdcc4324a612c6464531cda7 2024-11-12T06:49:40,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/b4f753b064b1416d8ce6f9daf78ef5d9 is 50, key is test_row_0/B:col10/1731394178959/Put/seqid=0 2024-11-12T06:49:40,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742298_1474 (size=12301) 2024-11-12T06:49:40,127 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/b4f753b064b1416d8ce6f9daf78ef5d9 2024-11-12T06:49:40,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/a6834c4ec7664fdda77ce4f56f5abe88 is 50, key is test_row_0/C:col10/1731394178959/Put/seqid=0 2024-11-12T06:49:40,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742299_1475 (size=12301) 2024-11-12T06:49:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on efaada0e985d01f31ce286661347290d 2024-11-12T06:49:40,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:40,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394240188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:40,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394240194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:40,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:40,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:40,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:40,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394240307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:40,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394240307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:40,505 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/a575a076060a4172937946fc5d91f201 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/a575a076060a4172937946fc5d91f201 2024-11-12T06:49:40,508 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efaada0e985d01f31ce286661347290d/B of efaada0e985d01f31ce286661347290d into a575a076060a4172937946fc5d91f201(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:40,508 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:40,508 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/B, priority=13, startTime=1731394180080; duration=0sec 2024-11-12T06:49:40,508 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:40,508 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:B 2024-11-12T06:49:40,508 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:40,509 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:40,509 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/C is initiating minor compaction (all files) 2024-11-12T06:49:40,509 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/C in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:40,509 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/e547c2d018444e1cbcae5a28e4b9c5e3, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/9a6392785bc648fa89f8269601fcb7a5, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/22de97d685bd4bc9af6db03e93b36f35] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=36.5 K 2024-11-12T06:49:40,510 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting e547c2d018444e1cbcae5a28e4b9c5e3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731394176931 2024-11-12T06:49:40,510 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a6392785bc648fa89f8269601fcb7a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731394177011 2024-11-12T06:49:40,510 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 22de97d685bd4bc9af6db03e93b36f35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731394178183 2024-11-12T06:49:40,512 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/9b859f0941f942da8fedbf4d4935f585 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/9b859f0941f942da8fedbf4d4935f585 2024-11-12T06:49:40,516 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#C#compaction#398 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:40,516 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efaada0e985d01f31ce286661347290d/A of efaada0e985d01f31ce286661347290d into 9b859f0941f942da8fedbf4d4935f585(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:40,516 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:40,516 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/A, priority=13, startTime=1731394180080; duration=0sec 2024-11-12T06:49:40,516 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:40,516 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:A 2024-11-12T06:49:40,517 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/2adb27a099ec49b8b9908fba76fb39c7 is 50, key is test_row_0/C:col10/1731394178842/Put/seqid=0 2024-11-12T06:49:40,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742300_1476 (size=12847) 2024-11-12T06:49:40,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:40,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394240527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:40,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:40,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:40,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:40,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394240527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:40,541 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/a6834c4ec7664fdda77ce4f56f5abe88 2024-11-12T06:49:40,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/ea0186fecdcc4324a612c6464531cda7 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/ea0186fecdcc4324a612c6464531cda7 2024-11-12T06:49:40,547 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/ea0186fecdcc4324a612c6464531cda7, entries=150, sequenceid=417, filesize=12.0 K 2024-11-12T06:49:40,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/b4f753b064b1416d8ce6f9daf78ef5d9 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/b4f753b064b1416d8ce6f9daf78ef5d9 2024-11-12T06:49:40,552 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/b4f753b064b1416d8ce6f9daf78ef5d9, entries=150, sequenceid=417, filesize=12.0 K 2024-11-12T06:49:40,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/a6834c4ec7664fdda77ce4f56f5abe88 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/a6834c4ec7664fdda77ce4f56f5abe88 2024-11-12T06:49:40,555 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/a6834c4ec7664fdda77ce4f56f5abe88, entries=150, sequenceid=417, filesize=12.0 K 2024-11-12T06:49:40,556 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for efaada0e985d01f31ce286661347290d in 466ms, sequenceid=417, compaction requested=false 2024-11-12T06:49:40,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:40,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:40,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-12T06:49:40,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-12T06:49:40,559 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-12T06:49:40,559 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6870 sec 2024-11-12T06:49:40,561 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.6920 sec 2024-11-12T06:49:40,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on efaada0e985d01f31ce286661347290d 2024-11-12T06:49:40,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efaada0e985d01f31ce286661347290d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T06:49:40,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=A 2024-11-12T06:49:40,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:40,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=B 2024-11-12T06:49:40,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:40,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=C 2024-11-12T06:49:40,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:40,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/288f90c263f54f60b2e0f7ab2a4a671b is 50, key is test_row_0/A:col10/1731394180854/Put/seqid=0 2024-11-12T06:49:40,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742301_1477 (size=14741) 2024-11-12T06:49:40,928 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/2adb27a099ec49b8b9908fba76fb39c7 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/2adb27a099ec49b8b9908fba76fb39c7 2024-11-12T06:49:40,931 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efaada0e985d01f31ce286661347290d/C of efaada0e985d01f31ce286661347290d into 2adb27a099ec49b8b9908fba76fb39c7(size=12.5 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:40,931 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:40,931 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/C, priority=13, startTime=1731394180080; duration=0sec 2024-11-12T06:49:40,931 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:40,932 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:C 2024-11-12T06:49:40,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394240922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:40,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394240930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:41,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T06:49:41,017 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:49:41,017 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:49:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-12T06:49:41,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T06:49:41,019 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:49:41,019 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:49:41,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:49:41,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:41,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394241047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:41,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:41,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394241047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T06:49:41,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:41,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:41,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:41,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:41,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:41,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:41,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:41,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:41,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394241227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:41,237 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_1' in efaada0e985d01f31ce286661347290d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:49:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:41,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:41,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394241237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:41,241 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_2' in efaada0e985d01f31ce286661347290d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:49:41,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:41,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:41,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:41,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:41,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394241247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:41,252 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_1' in efaada0e985d01f31ce286661347290d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:49:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:41,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/288f90c263f54f60b2e0f7ab2a4a671b 2024-11-12T06:49:41,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/cd71880a327f469a88501fc7a4be7e8c is 50, key is test_row_0/B:col10/1731394180854/Put/seqid=0 2024-11-12T06:49:41,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742302_1478 (size=12301) 2024-11-12T06:49:41,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:41,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394241267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:41,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:41,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:41,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394241267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:41,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:41,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:41,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:41,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:41,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:41,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:41,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:41,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:41,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:41,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:41,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T06:49:41,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:41,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:41,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:41,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:41,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:41,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:41,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:41,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:41,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394241577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:41,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:41,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:41,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:41,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394241587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:41,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:41,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:41,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:41,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:41,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:41,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:41,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:41,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T06:49:41,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/cd71880a327f469a88501fc7a4be7e8c 2024-11-12T06:49:41,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/7c60b06dd04a433aa12ac88761eeacb1 is 50, key is test_row_0/C:col10/1731394180854/Put/seqid=0 2024-11-12T06:49:41,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742303_1479 (size=12301) 2024-11-12T06:49:41,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:41,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:41,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:41,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:41,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:41,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:41,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:41,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:41,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:41,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:41,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:41,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:42,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/7c60b06dd04a433aa12ac88761eeacb1 2024-11-12T06:49:42,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/288f90c263f54f60b2e0f7ab2a4a671b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/288f90c263f54f60b2e0f7ab2a4a671b 2024-11-12T06:49:42,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/288f90c263f54f60b2e0f7ab2a4a671b, entries=200, sequenceid=431, filesize=14.4 K 2024-11-12T06:49:42,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/cd71880a327f469a88501fc7a4be7e8c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/cd71880a327f469a88501fc7a4be7e8c 2024-11-12T06:49:42,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:42,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:42,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:42,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:42,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:42,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:42,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:42,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/cd71880a327f469a88501fc7a4be7e8c, entries=150, sequenceid=431, filesize=12.0 K 2024-11-12T06:49:42,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/7c60b06dd04a433aa12ac88761eeacb1 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/7c60b06dd04a433aa12ac88761eeacb1 2024-11-12T06:49:42,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/7c60b06dd04a433aa12ac88761eeacb1, entries=150, sequenceid=431, filesize=12.0 K 2024-11-12T06:49:42,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for efaada0e985d01f31ce286661347290d in 1239ms, sequenceid=431, compaction requested=true 2024-11-12T06:49:42,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:42,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:49:42,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:42,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:49:42,095 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:42,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:42,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:49:42,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:42,095 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:42,095 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:42,095 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/B is initiating minor compaction (all files) 2024-11-12T06:49:42,095 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39787 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:42,096 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/B in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:42,096 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/A is initiating minor compaction (all files) 2024-11-12T06:49:42,096 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/A in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:42,096 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/a575a076060a4172937946fc5d91f201, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/b4f753b064b1416d8ce6f9daf78ef5d9, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/cd71880a327f469a88501fc7a4be7e8c] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=36.6 K 2024-11-12T06:49:42,096 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/9b859f0941f942da8fedbf4d4935f585, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/ea0186fecdcc4324a612c6464531cda7, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/288f90c263f54f60b2e0f7ab2a4a671b] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=38.9 K 2024-11-12T06:49:42,096 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b859f0941f942da8fedbf4d4935f585, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731394178183 2024-11-12T06:49:42,096 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting a575a076060a4172937946fc5d91f201, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731394178183 2024-11-12T06:49:42,096 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea0186fecdcc4324a612c6464531cda7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731394178959 2024-11-12T06:49:42,096 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting b4f753b064b1416d8ce6f9daf78ef5d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731394178959 2024-11-12T06:49:42,096 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 288f90c263f54f60b2e0f7ab2a4a671b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731394180174 2024-11-12T06:49:42,097 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting cd71880a327f469a88501fc7a4be7e8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731394180186 2024-11-12T06:49:42,102 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#B#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:42,103 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/20f5af66fcf449029d0a2c0232a320e1 is 50, key is test_row_0/B:col10/1731394180854/Put/seqid=0 2024-11-12T06:49:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on efaada0e985d01f31ce286661347290d 2024-11-12T06:49:42,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394180907, max timestamp=1731394182105Num uniques 859; 2024-11-12T06:49:42,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:49:42,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394180907, max timestamp=1731394182105Num uniques 859; 2024-11-12T06:49:42,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:49:42,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394180907, max timestamp=1731394182105Num uniques 859; 2024-11-12T06:49:42,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:49:42,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efaada0e985d01f31ce286661347290d 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-12T06:49:42,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=A 2024-11-12T06:49:42,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:42,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=B 2024-11-12T06:49:42,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:42,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=C 2024-11-12T06:49:42,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:42,113 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#A#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:42,113 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/06122280c929411ea60ff3758d83c401 is 50, key is test_row_0/A:col10/1731394180854/Put/seqid=0 2024-11-12T06:49:42,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/41b13b651a4e4df5921b58b881aa8efe is 50, key is test_row_0/A:col10/1731394180921/Put/seqid=0 2024-11-12T06:49:42,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742304_1480 (size=12949) 2024-11-12T06:49:42,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742305_1481 (size=14741) 2024-11-12T06:49:42,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742306_1482 (size=12847) 2024-11-12T06:49:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T06:49:42,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394182172, max timestamp=1731394182189Num uniques 859; 2024-11-12T06:49:42,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394182172, max timestamp=1731394182189Num uniques 859; 2024-11-12T06:49:42,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:42,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:42,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394182172, max timestamp=1731394182189Num uniques 859; 2024-11-12T06:49:42,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:42,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:42,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:42,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:42,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:42,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:42,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:42,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:42,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731394182226, max timestamp=1731394182261Num uniques 1068; 2024-11-12T06:49:42,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T06:49:42,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731394182226, max timestamp=1731394182261Num uniques 1068; 2024-11-12T06:49:42,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T06:49:42,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731394182226, max timestamp=1731394182261Num uniques 1068; 2024-11-12T06:49:42,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T06:49:42,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731394182298, max timestamp=1731394182359Num uniques 1077; 2024-11-12T06:49:42,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-12T06:49:42,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731394182298, max timestamp=1731394182359Num uniques 1077; 2024-11-12T06:49:42,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-12T06:49:42,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731394182298, max timestamp=1731394182359Num uniques 1077; 2024-11-12T06:49:42,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-12T06:49:42,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:42,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:42,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:42,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:42,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:42,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:42,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731394182433, max timestamp=1731394182449Num uniques 1136; 2024-11-12T06:49:42,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-12T06:49:42,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731394182433, max timestamp=1731394182449Num uniques 1136; 2024-11-12T06:49:42,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731394182433, max timestamp=1731394182449Num uniques 1136; 2024-11-12T06:49:42,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-12T06:49:42,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-12T06:49:42,525 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/20f5af66fcf449029d0a2c0232a320e1 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/20f5af66fcf449029d0a2c0232a320e1 2024-11-12T06:49:42,529 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efaada0e985d01f31ce286661347290d/B of efaada0e985d01f31ce286661347290d into 20f5af66fcf449029d0a2c0232a320e1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:42,529 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:42,529 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/B, priority=13, startTime=1731394182095; duration=0sec 2024-11-12T06:49:42,529 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:42,529 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:B 2024-11-12T06:49:42,529 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:42,530 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:42,530 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/C is initiating minor compaction (all files) 2024-11-12T06:49:42,530 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/C in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:42,530 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/2adb27a099ec49b8b9908fba76fb39c7, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/a6834c4ec7664fdda77ce4f56f5abe88, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/7c60b06dd04a433aa12ac88761eeacb1] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=36.6 K 2024-11-12T06:49:42,530 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 2adb27a099ec49b8b9908fba76fb39c7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731394178183 2024-11-12T06:49:42,530 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting a6834c4ec7664fdda77ce4f56f5abe88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731394178959 2024-11-12T06:49:42,531 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c60b06dd04a433aa12ac88761eeacb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731394180186 2024-11-12T06:49:42,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/41b13b651a4e4df5921b58b881aa8efe 2024-11-12T06:49:42,536 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#C#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:42,536 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/7ba6e074210445cc977b10a5dfecc62b is 50, key is test_row_0/C:col10/1731394180854/Put/seqid=0 2024-11-12T06:49:42,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/1a6c0405ec5a4dbeb8ad6b5731e21705 is 50, key is test_row_0/B:col10/1731394180921/Put/seqid=0 2024-11-12T06:49:42,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742307_1483 (size=12949) 2024-11-12T06:49:42,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:42,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:42,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:42,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:42,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:42,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:42,547 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/06122280c929411ea60ff3758d83c401 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/06122280c929411ea60ff3758d83c401 2024-11-12T06:49:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:42,553 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efaada0e985d01f31ce286661347290d/A of efaada0e985d01f31ce286661347290d into 06122280c929411ea60ff3758d83c401(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:42,553 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:42,553 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/A, priority=13, startTime=1731394182095; duration=0sec 2024-11-12T06:49:42,553 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:42,553 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:A 2024-11-12T06:49:42,555 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/7ba6e074210445cc977b10a5dfecc62b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/7ba6e074210445cc977b10a5dfecc62b 2024-11-12T06:49:42,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742308_1484 (size=12301) 2024-11-12T06:49:42,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731394182508, max timestamp=1731394182556Num uniques 1095; 2024-11-12T06:49:42,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-12T06:49:42,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731394182508, max timestamp=1731394182556Num uniques 1095; 2024-11-12T06:49:42,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-12T06:49:42,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731394182508, max timestamp=1731394182556Num uniques 1095; 2024-11-12T06:49:42,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-12T06:49:42,561 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efaada0e985d01f31ce286661347290d/C of efaada0e985d01f31ce286661347290d into 7ba6e074210445cc977b10a5dfecc62b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:42,561 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:42,561 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/C, priority=13, startTime=1731394182095; duration=0sec 2024-11-12T06:49:42,561 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:42,561 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:C 2024-11-12T06:49:42,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731394182618, max timestamp=1731394182640Num uniques 1104; 2024-11-12T06:49:42,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-12T06:49:42,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731394182618, max timestamp=1731394182640Num uniques 1104; 2024-11-12T06:49:42,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731394182618, max timestamp=1731394182640Num uniques 1104; 2024-11-12T06:49:42,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-12T06:49:42,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-12T06:49:42,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:42,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:42,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:42,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:42,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:42,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:42,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731394182655, max timestamp=1731394182717Num uniques 1063; 2024-11-12T06:49:42,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731394182655, max timestamp=1731394182717Num uniques 1063; 2024-11-12T06:49:42,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-12T06:49:42,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731394182655, max timestamp=1731394182717Num uniques 1063; 2024-11-12T06:49:42,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-12T06:49:42,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-12T06:49:42,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731394182741, max timestamp=1731394182785Num uniques 1072; 2024-11-12T06:49:42,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-12T06:49:42,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731394182741, max timestamp=1731394182785Num uniques 1022; 2024-11-12T06:49:42,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731394182741, max timestamp=1731394182785Num uniques 1022; 2024-11-12T06:49:42,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-12T06:49:42,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-12T06:49:42,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:42,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:42,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:42,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:42,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:42,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:42,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731394182829, max timestamp=1731394182854Num uniques 1081; 2024-11-12T06:49:42,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-12T06:49:42,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731394182829, max timestamp=1731394182854Num uniques 1081; 2024-11-12T06:49:42,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-12T06:49:42,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731394182829, max timestamp=1731394182854Num uniques 1081; 2024-11-12T06:49:42,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-12T06:49:42,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/1a6c0405ec5a4dbeb8ad6b5731e21705 2024-11-12T06:49:42,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/c73622740bac43079c6ab1bb36757464 is 50, key is test_row_0/C:col10/1731394180921/Put/seqid=0 2024-11-12T06:49:42,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742309_1485 (size=12301) 2024-11-12T06:49:43,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:43,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:43,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:43,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:43,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:43,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:43,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:43,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731394182977, max timestamp=1731394183005Num uniques 1090; 2024-11-12T06:49:43,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-12T06:49:43,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731394182977, max timestamp=1731394183005Num uniques 1090; 2024-11-12T06:49:43,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-12T06:49:43,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731394182977, max timestamp=1731394183005Num uniques 1090; 2024-11-12T06:49:43,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-12T06:49:43,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731394183059, max timestamp=1731394183075Num uniques 1099; 2024-11-12T06:49:43,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-12T06:49:43,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731394183059, max timestamp=1731394183075Num uniques 1099; 2024-11-12T06:49:43,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-12T06:49:43,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731394183059, max timestamp=1731394183075Num uniques 1099; 2024-11-12T06:49:43,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-12T06:49:43,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731394183129, max timestamp=1731394183144Num uniques 1108; 2024-11-12T06:49:43,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-12T06:49:43,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731394183129, max timestamp=1731394183144Num uniques 1108; 2024-11-12T06:49:43,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731394183129, max timestamp=1731394183144Num uniques 1108; 2024-11-12T06:49:43,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-12T06:49:43,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-12T06:49:43,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:43,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:43,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:43,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:43,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:43,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T06:49:43,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731394183189, max timestamp=1731394183214Num uniques 1067; 2024-11-12T06:49:43,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-12T06:49:43,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731394183189, max timestamp=1731394183214Num uniques 1067; 2024-11-12T06:49:43,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-12T06:49:43,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731394183189, max timestamp=1731394183214Num uniques 1067; 2024-11-12T06:49:43,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-12T06:49:43,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731394183264, max timestamp=1731394183283Num uniques 1076; 2024-11-12T06:49:43,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731394183264, max timestamp=1731394183283Num uniques 1076; 2024-11-12T06:49:43,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-12T06:49:43,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-12T06:49:43,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731394183264, max timestamp=1731394183283Num uniques 1076; 2024-11-12T06:49:43,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-12T06:49:43,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:43,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:43,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:43,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:43,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:43,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:43,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:43,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=335, cellSize=15340, totalHeapSize=35080, min timestamp=1731394183320, max timestamp=1731394183349Num uniques 1085; 2024-11-12T06:49:43,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=15340, suffix heap size=140808,new segment heap size=35080 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=335 2024-11-12T06:49:43,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=335, cellSize=15340, totalHeapSize=35080, min timestamp=1731394183320, max timestamp=1731394183349Num uniques 1085; 2024-11-12T06:49:43,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=15340, suffix heap size=140808,new segment heap size=35080 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=335 2024-11-12T06:49:43,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=335, cellSize=15340, totalHeapSize=35080, min timestamp=1731394183320, max timestamp=1731394183349Num uniques 1085; 2024-11-12T06:49:43,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=15340, suffix heap size=140808,new segment heap size=35080 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=335 2024-11-12T06:49:43,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/c73622740bac43079c6ab1bb36757464 2024-11-12T06:49:43,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/41b13b651a4e4df5921b58b881aa8efe as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/41b13b651a4e4df5921b58b881aa8efe 2024-11-12T06:49:43,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/41b13b651a4e4df5921b58b881aa8efe, entries=200, sequenceid=453, filesize=14.4 K 2024-11-12T06:49:43,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/1a6c0405ec5a4dbeb8ad6b5731e21705 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/1a6c0405ec5a4dbeb8ad6b5731e21705 2024-11-12T06:49:43,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/1a6c0405ec5a4dbeb8ad6b5731e21705, entries=150, sequenceid=453, filesize=12.0 K 2024-11-12T06:49:43,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/c73622740bac43079c6ab1bb36757464 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/c73622740bac43079c6ab1bb36757464 2024-11-12T06:49:43,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/c73622740bac43079c6ab1bb36757464, entries=150, sequenceid=453, filesize=12.0 K 2024-11-12T06:49:43,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=87.22 KB/89310 for efaada0e985d01f31ce286661347290d in 1270ms, sequenceid=453, compaction requested=false 2024-11-12T06:49:43,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:43,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on efaada0e985d01f31ce286661347290d 2024-11-12T06:49:43,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efaada0e985d01f31ce286661347290d 3/3 column families, dataSize=93.93 KB heapSize=248.95 KB 2024-11-12T06:49:43,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=A 2024-11-12T06:49:43,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:43,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=B 2024-11-12T06:49:43,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:43,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=C 2024-11-12T06:49:43,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:43,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/5eadaaa85bc84355bb1c9188d99125a2 is 50, key is test_row_0/A:col10/1731394183369/Put/seqid=0 2024-11-12T06:49:43,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742310_1486 (size=14741) 2024-11-12T06:49:43,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:43,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394243437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:43,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:43,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:43,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:43,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394243445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:43,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:43,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:43,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:43,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:43,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:43,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:43,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:43,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:43,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394243557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:43,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:43,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:43,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 311 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394243557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:43,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:43,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:43,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:43,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:43,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:43,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:43,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:43,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:43,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:43,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:43,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:43,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:43,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:43,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:43,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:43,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:43,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:43,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:43,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394243777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:43,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:43,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:43,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 312 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394243777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:43,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:43,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:43,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:43,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:43,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=724 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/5eadaaa85bc84355bb1c9188d99125a2 2024-11-12T06:49:43,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/123d42f204f24ef896110221926f4430 is 50, key is test_row_0/B:col10/1731394183369/Put/seqid=0 2024-11-12T06:49:43,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742311_1487 (size=12301) 2024-11-12T06:49:43,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:43,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:43,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:43,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:43,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:43,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:44,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:44,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:44,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:44,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:44,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:44,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:44,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394244086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:44,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 313 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394244087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:44,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=724 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/123d42f204f24ef896110221926f4430 2024-11-12T06:49:44,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/52ea2859fa0146da9d7a8533326faf56 is 50, key is test_row_0/C:col10/1731394183369/Put/seqid=0 2024-11-12T06:49:44,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742312_1488 (size=12301) 2024-11-12T06:49:44,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:44,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:44,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:44,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:44,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:44,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:44,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:44,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:44,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:44,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:44,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:44,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:44,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:44,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:44,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:44,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:44,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:44,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:44,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 314 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394244597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:44,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394244597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:44,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=724 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/52ea2859fa0146da9d7a8533326faf56 2024-11-12T06:49:44,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/5eadaaa85bc84355bb1c9188d99125a2 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/5eadaaa85bc84355bb1c9188d99125a2 2024-11-12T06:49:44,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/5eadaaa85bc84355bb1c9188d99125a2, entries=200, sequenceid=724, filesize=14.4 K 2024-11-12T06:49:44,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/123d42f204f24ef896110221926f4430 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/123d42f204f24ef896110221926f4430 2024-11-12T06:49:44,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/123d42f204f24ef896110221926f4430, entries=150, sequenceid=724, filesize=12.0 K 2024-11-12T06:49:44,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/52ea2859fa0146da9d7a8533326faf56 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/52ea2859fa0146da9d7a8533326faf56 2024-11-12T06:49:44,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/52ea2859fa0146da9d7a8533326faf56, entries=150, sequenceid=724, filesize=12.0 K 2024-11-12T06:49:44,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~248.91 KB/254880, currentSize=93.93 KB/96180 for efaada0e985d01f31ce286661347290d in 1239ms, sequenceid=724, compaction requested=true 2024-11-12T06:49:44,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:44,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:49:44,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:44,624 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:44,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:49:44,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:44,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:49:44,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:44,624 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:44,625 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:44,625 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42329 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:44,625 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/A is initiating minor compaction (all files) 2024-11-12T06:49:44,625 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/B is initiating minor compaction (all files) 2024-11-12T06:49:44,625 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/A in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:44,625 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/B in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:44,625 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/06122280c929411ea60ff3758d83c401, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/41b13b651a4e4df5921b58b881aa8efe, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/5eadaaa85bc84355bb1c9188d99125a2] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=41.3 K 2024-11-12T06:49:44,625 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/20f5af66fcf449029d0a2c0232a320e1, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/1a6c0405ec5a4dbeb8ad6b5731e21705, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/123d42f204f24ef896110221926f4430] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=36.7 K 2024-11-12T06:49:44,625 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 20f5af66fcf449029d0a2c0232a320e1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731394180186 2024-11-12T06:49:44,625 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06122280c929411ea60ff3758d83c401, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731394180186 2024-11-12T06:49:44,625 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41b13b651a4e4df5921b58b881aa8efe, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731394180907 2024-11-12T06:49:44,625 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a6c0405ec5a4dbeb8ad6b5731e21705, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731394180921 2024-11-12T06:49:44,625 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5eadaaa85bc84355bb1c9188d99125a2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=724, earliestPutTs=1731394183369 2024-11-12T06:49:44,625 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 123d42f204f24ef896110221926f4430, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=724, earliestPutTs=1731394183369 2024-11-12T06:49:44,630 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#A#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:44,631 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/e552594cc81b48efb410aafed15669da is 50, key is test_row_0/A:col10/1731394183369/Put/seqid=0 2024-11-12T06:49:44,631 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#B#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:44,631 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/90df3fed6dbf4b7ab06b354f5e74cb92 is 50, key is test_row_0/B:col10/1731394183369/Put/seqid=0 2024-11-12T06:49:44,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742314_1490 (size=13051) 2024-11-12T06:49:44,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742313_1489 (size=12949) 2024-11-12T06:49:44,654 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/90df3fed6dbf4b7ab06b354f5e74cb92 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/90df3fed6dbf4b7ab06b354f5e74cb92 2024-11-12T06:49:44,655 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/e552594cc81b48efb410aafed15669da as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/e552594cc81b48efb410aafed15669da 2024-11-12T06:49:44,659 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efaada0e985d01f31ce286661347290d/A of efaada0e985d01f31ce286661347290d into e552594cc81b48efb410aafed15669da(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:44,659 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efaada0e985d01f31ce286661347290d/B of efaada0e985d01f31ce286661347290d into 90df3fed6dbf4b7ab06b354f5e74cb92(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:44,659 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:44,659 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:44,659 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/A, priority=13, startTime=1731394184624; duration=0sec 2024-11-12T06:49:44,659 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/B, priority=13, startTime=1731394184624; duration=0sec 2024-11-12T06:49:44,659 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:44,659 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:44,659 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:B 2024-11-12T06:49:44,659 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:A 2024-11-12T06:49:44,660 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:44,664 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:44,664 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/C is initiating minor compaction (all files) 2024-11-12T06:49:44,664 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/C in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:44,664 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/7ba6e074210445cc977b10a5dfecc62b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/c73622740bac43079c6ab1bb36757464, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/52ea2859fa0146da9d7a8533326faf56] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=36.7 K 2024-11-12T06:49:44,666 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ba6e074210445cc977b10a5dfecc62b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731394180186 2024-11-12T06:49:44,666 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting c73622740bac43079c6ab1bb36757464, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731394180921 2024-11-12T06:49:44,666 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 52ea2859fa0146da9d7a8533326faf56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=724, earliestPutTs=1731394183369 2024-11-12T06:49:44,674 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#C#compaction#413 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:44,674 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/09077310b8bf4043aacd8ca122d0327f is 50, key is test_row_0/C:col10/1731394183369/Put/seqid=0 2024-11-12T06:49:44,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T06:49:44,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:44,681 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing efaada0e985d01f31ce286661347290d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T06:49:44,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=A 2024-11-12T06:49:44,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:44,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=B 2024-11-12T06:49:44,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:44,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=C 2024-11-12T06:49:44,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:44,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742315_1491 (size=13051) 2024-11-12T06:49:44,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/a10325430d6e45d38cad3da7b2646942 is 50, key is test_row_0/A:col10/1731394183415/Put/seqid=0 2024-11-12T06:49:44,690 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/09077310b8bf4043aacd8ca122d0327f as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/09077310b8bf4043aacd8ca122d0327f 2024-11-12T06:49:44,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742316_1492 (size=12301) 2024-11-12T06:49:44,693 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=743 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/a10325430d6e45d38cad3da7b2646942 2024-11-12T06:49:44,695 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efaada0e985d01f31ce286661347290d/C of efaada0e985d01f31ce286661347290d into 09077310b8bf4043aacd8ca122d0327f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:44,695 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:44,695 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/C, priority=13, startTime=1731394184624; duration=0sec 2024-11-12T06:49:44,695 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:44,695 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:C 2024-11-12T06:49:44,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/eef7a77402ae45e88d5fbd3997509190 is 50, key is test_row_0/B:col10/1731394183415/Put/seqid=0 2024-11-12T06:49:44,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742317_1493 (size=12301) 2024-11-12T06:49:45,109 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=743 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/eef7a77402ae45e88d5fbd3997509190 2024-11-12T06:49:45,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/087bfe85974b45be8841f41917f6c8c5 is 50, key is test_row_0/C:col10/1731394183415/Put/seqid=0 2024-11-12T06:49:45,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742318_1494 (size=12301) 2024-11-12T06:49:45,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T06:49:45,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on efaada0e985d01f31ce286661347290d 2024-11-12T06:49:45,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. as already flushing 2024-11-12T06:49:45,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:45,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394245285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:45,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:45,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:45,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394245291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:45,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:45,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394245292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:45,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:45,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394245397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:45,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:45,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394245398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:45,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:45,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:45,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:45,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394245407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:45,518 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=743 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/087bfe85974b45be8841f41917f6c8c5 2024-11-12T06:49:45,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/a10325430d6e45d38cad3da7b2646942 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/a10325430d6e45d38cad3da7b2646942 2024-11-12T06:49:45,524 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/a10325430d6e45d38cad3da7b2646942, entries=150, sequenceid=743, filesize=12.0 K 2024-11-12T06:49:45,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/eef7a77402ae45e88d5fbd3997509190 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/eef7a77402ae45e88d5fbd3997509190 2024-11-12T06:49:45,528 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/eef7a77402ae45e88d5fbd3997509190, entries=150, sequenceid=743, filesize=12.0 K 2024-11-12T06:49:45,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/087bfe85974b45be8841f41917f6c8c5 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/087bfe85974b45be8841f41917f6c8c5 2024-11-12T06:49:45,531 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/087bfe85974b45be8841f41917f6c8c5, entries=150, sequenceid=743, filesize=12.0 K 2024-11-12T06:49:45,532 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for efaada0e985d01f31ce286661347290d in 851ms, sequenceid=743, compaction requested=false 2024-11-12T06:49:45,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:45,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:45,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-12T06:49:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-12T06:49:45,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-12T06:49:45,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.5130 sec 2024-11-12T06:49:45,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 4.5180 sec 2024-11-12T06:49:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on efaada0e985d01f31ce286661347290d 2024-11-12T06:49:45,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efaada0e985d01f31ce286661347290d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T06:49:45,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=A 2024-11-12T06:49:45,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:45,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=B 2024-11-12T06:49:45,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:45,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=C 2024-11-12T06:49:45,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:45,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/9de5a04e8b4646e9a783539b28c7478b is 50, key is test_row_0/A:col10/1731394185280/Put/seqid=0 2024-11-12T06:49:45,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742319_1495 (size=14741) 2024-11-12T06:49:45,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 317 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394245636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:45,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:45,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394245637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:45,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:45,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394245638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:45,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:45,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:45,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394245639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:45,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:45,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:45,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394245641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:45,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:45,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 318 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394245747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:45,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394245757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:45,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394245757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:45,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394245757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394245757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:45,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 319 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394245957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:45,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394245967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394245967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:45,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:45,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394245977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:45,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:45,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394245978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:45,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:45,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:45,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:46,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=760 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/9de5a04e8b4646e9a783539b28c7478b 2024-11-12T06:49:46,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/caa15e69b51e4300a9151d8a3fccc93c is 50, key is test_row_0/B:col10/1731394185280/Put/seqid=0 2024-11-12T06:49:46,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742320_1496 (size=12301) 2024-11-12T06:49:46,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 320 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394246267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:46,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:46,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394246287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:46,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394246287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:46,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:46,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:46,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394246297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:46,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:46,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394246297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:46,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:46,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:46,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=760 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/caa15e69b51e4300a9151d8a3fccc93c 2024-11-12T06:49:46,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/f548f455b1694f5cb011f05793aae072 is 50, key is test_row_0/C:col10/1731394185280/Put/seqid=0 2024-11-12T06:49:46,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742321_1497 (size=12301) 2024-11-12T06:49:46,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=760 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/f548f455b1694f5cb011f05793aae072 2024-11-12T06:49:46,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/9de5a04e8b4646e9a783539b28c7478b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/9de5a04e8b4646e9a783539b28c7478b 2024-11-12T06:49:46,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/9de5a04e8b4646e9a783539b28c7478b, entries=200, sequenceid=760, filesize=14.4 K 2024-11-12T06:49:46,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/caa15e69b51e4300a9151d8a3fccc93c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/caa15e69b51e4300a9151d8a3fccc93c 2024-11-12T06:49:46,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/caa15e69b51e4300a9151d8a3fccc93c, entries=150, sequenceid=760, filesize=12.0 K 2024-11-12T06:49:46,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/f548f455b1694f5cb011f05793aae072 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/f548f455b1694f5cb011f05793aae072 2024-11-12T06:49:46,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/f548f455b1694f5cb011f05793aae072, entries=150, sequenceid=760, filesize=12.0 K 2024-11-12T06:49:46,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for efaada0e985d01f31ce286661347290d in 860ms, sequenceid=760, compaction requested=true 2024-11-12T06:49:46,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:46,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:49:46,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:46,470 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:46,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:49:46,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:46,470 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:46,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:49:46,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:46,471 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:46,471 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:46,471 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/A is initiating minor compaction (all files) 2024-11-12T06:49:46,471 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/B is initiating minor compaction (all files) 2024-11-12T06:49:46,471 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/A in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:46,471 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/B in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:46,471 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/e552594cc81b48efb410aafed15669da, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/a10325430d6e45d38cad3da7b2646942, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/9de5a04e8b4646e9a783539b28c7478b] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=39.1 K 2024-11-12T06:49:46,471 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/90df3fed6dbf4b7ab06b354f5e74cb92, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/eef7a77402ae45e88d5fbd3997509190, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/caa15e69b51e4300a9151d8a3fccc93c] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=36.8 K 2024-11-12T06:49:46,471 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting e552594cc81b48efb410aafed15669da, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=724, earliestPutTs=1731394183369 2024-11-12T06:49:46,471 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 90df3fed6dbf4b7ab06b354f5e74cb92, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=724, earliestPutTs=1731394183369 2024-11-12T06:49:46,471 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting a10325430d6e45d38cad3da7b2646942, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=743, earliestPutTs=1731394183415 2024-11-12T06:49:46,471 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting eef7a77402ae45e88d5fbd3997509190, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=743, earliestPutTs=1731394183415 2024-11-12T06:49:46,472 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9de5a04e8b4646e9a783539b28c7478b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=760, earliestPutTs=1731394185280 2024-11-12T06:49:46,472 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting caa15e69b51e4300a9151d8a3fccc93c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=760, earliestPutTs=1731394185280 2024-11-12T06:49:46,477 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#A#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:46,478 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#B#compaction#421 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:46,478 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/3a41506ba08745c59c2f1a0eb345c6ce is 50, key is test_row_0/A:col10/1731394185280/Put/seqid=0 2024-11-12T06:49:46,478 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/188b3d1d961c4ecc8248740743342395 is 50, key is test_row_0/B:col10/1731394185280/Put/seqid=0 2024-11-12T06:49:46,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742323_1499 (size=13153) 2024-11-12T06:49:46,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742322_1498 (size=13051) 2024-11-12T06:49:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on efaada0e985d01f31ce286661347290d 2024-11-12T06:49:46,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efaada0e985d01f31ce286661347290d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T06:49:46,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=A 2024-11-12T06:49:46,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:46,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=B 2024-11-12T06:49:46,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:46,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=C 2024-11-12T06:49:46,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:46,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/b7a7e6a5a60445d783abbffec9ed1bff is 50, key is test_row_0/A:col10/1731394185637/Put/seqid=0 2024-11-12T06:49:46,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742324_1500 (size=14741) 2024-11-12T06:49:46,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394246827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:46,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394246828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:46,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394246835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:46,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394246840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:46,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:46,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394246842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:46,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:46,902 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/188b3d1d961c4ecc8248740743342395 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/188b3d1d961c4ecc8248740743342395 2024-11-12T06:49:46,904 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/3a41506ba08745c59c2f1a0eb345c6ce as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/3a41506ba08745c59c2f1a0eb345c6ce 2024-11-12T06:49:46,907 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efaada0e985d01f31ce286661347290d/B of efaada0e985d01f31ce286661347290d into 188b3d1d961c4ecc8248740743342395(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:46,907 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:46,907 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/B, priority=13, startTime=1731394186470; duration=0sec 2024-11-12T06:49:46,908 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:49:46,908 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:B 2024-11-12T06:49:46,908 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:49:46,909 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:49:46,910 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): efaada0e985d01f31ce286661347290d/C is initiating minor compaction (all files) 2024-11-12T06:49:46,910 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efaada0e985d01f31ce286661347290d/C in TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:46,910 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/09077310b8bf4043aacd8ca122d0327f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/087bfe85974b45be8841f41917f6c8c5, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/f548f455b1694f5cb011f05793aae072] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp, totalSize=36.8 K 2024-11-12T06:49:46,910 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 09077310b8bf4043aacd8ca122d0327f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=724, earliestPutTs=1731394183369 2024-11-12T06:49:46,911 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 087bfe85974b45be8841f41917f6c8c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=743, earliestPutTs=1731394183415 2024-11-12T06:49:46,911 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting f548f455b1694f5cb011f05793aae072, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=760, earliestPutTs=1731394185280 2024-11-12T06:49:46,911 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efaada0e985d01f31ce286661347290d/A of efaada0e985d01f31ce286661347290d into 3a41506ba08745c59c2f1a0eb345c6ce(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:46,911 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:46,912 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/A, priority=13, startTime=1731394186470; duration=0sec 2024-11-12T06:49:46,912 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:46,912 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:A 2024-11-12T06:49:46,916 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efaada0e985d01f31ce286661347290d#C#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:49:46,916 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/66ef588f25444303b868a7868757029d is 50, key is test_row_0/C:col10/1731394185280/Put/seqid=0 2024-11-12T06:49:46,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742325_1501 (size=13153) 2024-11-12T06:49:46,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:46,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394246947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:46,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:46,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:46,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394246947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:46,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:46,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:46,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 327 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394246947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:46,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:46,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:46,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394246957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:46,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:46,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394246957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:46,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:46,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:46,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:47,109 INFO [Thread-1918 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1918. 2024-11-12T06:49:47,109 DEBUG [Thread-1918 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:49:47,109 DEBUG [Thread-1918 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:47,109 INFO [Thread-1916 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1916. 2024-11-12T06:49:47,109 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:49:47,109 DEBUG [Thread-1916 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:49:47,109 DEBUG [Thread-1916 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:47,110 DEBUG [Thread-1916 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:47,110 DEBUG [Thread-1918 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:47,110 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:49:47,112 INFO [Thread-1912 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1912. 2024-11-12T06:49:47,112 DEBUG [Thread-1912 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:49:47,112 DEBUG [Thread-1912 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:47,112 DEBUG [Thread-1912 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:47,112 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:49:47,120 INFO [Thread-1920 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1920. 2024-11-12T06:49:47,120 DEBUG [Thread-1920 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:49:47,120 DEBUG [Thread-1920 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:47,120 DEBUG [Thread-1920 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:47,120 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:49:47,121 INFO [Thread-1914 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1914. 2024-11-12T06:49:47,121 DEBUG [Thread-1914 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:49:47,121 DEBUG [Thread-1914 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:47,121 DEBUG [Thread-1914 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:47,121 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:49:47,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:47,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:47,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:47,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394247157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:47,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 328 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394247157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:47,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394247157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:47,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:47,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394247177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394247177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:47,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:47,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:47,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=777 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/b7a7e6a5a60445d783abbffec9ed1bff 2024-11-12T06:49:47,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/f2359636fb2c4b719ca2c3a33e1a7b39 is 50, key is test_row_0/B:col10/1731394185637/Put/seqid=0 2024-11-12T06:49:47,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742326_1502 (size=12301) 2024-11-12T06:49:47,329 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/66ef588f25444303b868a7868757029d as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/66ef588f25444303b868a7868757029d 2024-11-12T06:49:47,336 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efaada0e985d01f31ce286661347290d/C of efaada0e985d01f31ce286661347290d into 66ef588f25444303b868a7868757029d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:49:47,336 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:47,336 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., storeName=efaada0e985d01f31ce286661347290d/C, priority=13, startTime=1731394186470; duration=0sec 2024-11-12T06:49:47,337 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:47,337 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:C 2024-11-12T06:49:47,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:47,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:47,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394247468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 329 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394247468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394247468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:47,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:47,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:47,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:47,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394247488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:47,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394247488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:47,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=777 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/f2359636fb2c4b719ca2c3a33e1a7b39 2024-11-12T06:49:47,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/5e75327637f54bb8a2099fdc10c2fc75 is 50, key is test_row_0/C:col10/1731394185637/Put/seqid=0 2024-11-12T06:49:47,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742327_1503 (size=12301) 2024-11-12T06:49:47,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:47,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:47,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60230 deadline: 1731394247978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 330 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60286 deadline: 1731394247978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731394247978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:47,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:47,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:47,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:47,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:47,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60270 deadline: 1731394247998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:47,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 294 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1731394247998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:48,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:48,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:48,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:48,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 , the old value is region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:48,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:48,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=2 because the exception is null or not the one we care about 2024-11-12T06:49:48,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=777 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/5e75327637f54bb8a2099fdc10c2fc75 2024-11-12T06:49:48,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/b7a7e6a5a60445d783abbffec9ed1bff as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/b7a7e6a5a60445d783abbffec9ed1bff 2024-11-12T06:49:48,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/b7a7e6a5a60445d783abbffec9ed1bff, entries=200, sequenceid=777, filesize=14.4 K 2024-11-12T06:49:48,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/f2359636fb2c4b719ca2c3a33e1a7b39 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/f2359636fb2c4b719ca2c3a33e1a7b39 2024-11-12T06:49:48,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/f2359636fb2c4b719ca2c3a33e1a7b39, entries=150, sequenceid=777, filesize=12.0 K 2024-11-12T06:49:48,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/5e75327637f54bb8a2099fdc10c2fc75 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/5e75327637f54bb8a2099fdc10c2fc75 2024-11-12T06:49:48,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/5e75327637f54bb8a2099fdc10c2fc75, entries=150, sequenceid=777, filesize=12.0 K 2024-11-12T06:49:48,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for efaada0e985d01f31ce286661347290d in 1260ms, sequenceid=777, compaction requested=false 2024-11-12T06:49:48,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on efaada0e985d01f31ce286661347290d 2024-11-12T06:49:48,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efaada0e985d01f31ce286661347290d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T06:49:48,989 INFO [Thread-1903 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1903. 2024-11-12T06:49:48,989 DEBUG [Thread-1903 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:49:48,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=A 2024-11-12T06:49:48,989 DEBUG [Thread-1903 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:48,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:48,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=B 2024-11-12T06:49:48,989 DEBUG [Thread-1903 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:48,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:48,989 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:49:48,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=C 2024-11-12T06:49:48,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:48,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/cf836aeb8c2d4149a42d3820eee5ec1e is 50, key is test_row_0/A:col10/1731394188988/Put/seqid=0 2024-11-12T06:49:48,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742328_1504 (size=12301) 2024-11-12T06:49:48,998 INFO [Thread-1901 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1901. 2024-11-12T06:49:48,998 INFO [Thread-1907 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1907. 2024-11-12T06:49:48,998 DEBUG [Thread-1901 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:49:48,998 DEBUG [Thread-1901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:48,998 DEBUG [Thread-1907 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:49:48,998 DEBUG [Thread-1907 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:48,999 DEBUG [Thread-1901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:48,999 DEBUG [Thread-1907 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:48,999 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:49:48,999 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:49:49,009 INFO [Thread-1909 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1909. 2024-11-12T06:49:49,009 INFO [Thread-1905 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1905. 2024-11-12T06:49:49,009 DEBUG [Thread-1905 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:49:49,009 DEBUG [Thread-1909 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:49:49,009 DEBUG [Thread-1905 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:49,009 DEBUG [Thread-1909 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:49,009 DEBUG [Thread-1905 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:49,009 DEBUG [Thread-1909 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:49,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:49:49,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:49:49,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T06:49:49,218 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:49:49,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T06:49:49,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 276 2024-11-12T06:49:49,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-12T06:49:49,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-12T06:49:49,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-12T06:49:49,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 238 2024-11-12T06:49:49,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T06:49:49,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T06:49:49,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1217 2024-11-12T06:49:49,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3647 rows 2024-11-12T06:49:49,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1213 2024-11-12T06:49:49,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3639 rows 2024-11-12T06:49:49,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1217 2024-11-12T06:49:49,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3650 rows 2024-11-12T06:49:49,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1220 2024-11-12T06:49:49,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3652 rows 2024-11-12T06:49:49,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1217 2024-11-12T06:49:49,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3647 rows 2024-11-12T06:49:49,219 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T06:49:49,220 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T06:49:49,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:49,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:49,220 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:49:49,222 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T06:49:49,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T06:49:49,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T06:49:49,229 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394189228"}]},"ts":"1731394189228"} 2024-11-12T06:49:49,231 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T06:49:49,231 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T06:49:49,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T06:49:49,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efaada0e985d01f31ce286661347290d, UNASSIGN}] 2024-11-12T06:49:49,233 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efaada0e985d01f31ce286661347290d, UNASSIGN 2024-11-12T06:49:49,233 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=efaada0e985d01f31ce286661347290d, regionState=CLOSING, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:49,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=efaada0e985d01f31ce286661347290d, UNASSIGN because future has completed 2024-11-12T06:49:49,235 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T06:49:49,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622}] 2024-11-12T06:49:49,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T06:49:49,388 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close efaada0e985d01f31ce286661347290d 2024-11-12T06:49:49,388 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T06:49:49,388 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing efaada0e985d01f31ce286661347290d, disabling compactions & flushes 2024-11-12T06:49:49,388 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:49,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=797 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/cf836aeb8c2d4149a42d3820eee5ec1e 2024-11-12T06:49:49,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/da12976ea3504696bf102a4db0f66c20 is 50, key is test_row_0/B:col10/1731394188988/Put/seqid=0 2024-11-12T06:49:49,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742329_1505 (size=12301) 2024-11-12T06:49:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T06:49:49,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=797 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/da12976ea3504696bf102a4db0f66c20 2024-11-12T06:49:49,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/523250a62d0a4fe6834071cf24c6ee79 is 50, key is test_row_0/C:col10/1731394188988/Put/seqid=0 2024-11-12T06:49:49,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742330_1506 (size=12301) 2024-11-12T06:49:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T06:49:50,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=797 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/523250a62d0a4fe6834071cf24c6ee79 2024-11-12T06:49:50,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/cf836aeb8c2d4149a42d3820eee5ec1e as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/cf836aeb8c2d4149a42d3820eee5ec1e 2024-11-12T06:49:50,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/cf836aeb8c2d4149a42d3820eee5ec1e, entries=150, sequenceid=797, filesize=12.0 K 2024-11-12T06:49:50,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/da12976ea3504696bf102a4db0f66c20 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/da12976ea3504696bf102a4db0f66c20 2024-11-12T06:49:50,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/da12976ea3504696bf102a4db0f66c20, entries=150, sequenceid=797, filesize=12.0 K 2024-11-12T06:49:50,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/523250a62d0a4fe6834071cf24c6ee79 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/523250a62d0a4fe6834071cf24c6ee79 2024-11-12T06:49:50,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/523250a62d0a4fe6834071cf24c6ee79, entries=150, sequenceid=797, filesize=12.0 K 2024-11-12T06:49:50,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=26.84 KB/27480 for efaada0e985d01f31ce286661347290d in 1268ms, sequenceid=797, compaction requested=true 2024-11-12T06:49:50,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efaada0e985d01f31ce286661347290d: 2024-11-12T06:49:50,257 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:50,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:49:50,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:50,258 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. because compaction request was cancelled 2024-11-12T06:49:50,258 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:50,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:49:50,258 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:A 2024-11-12T06:49:50,258 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. after waiting 0 ms 2024-11-12T06:49:50,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:50,258 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. because compaction request was cancelled 2024-11-12T06:49:50,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efaada0e985d01f31ce286661347290d:C, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:49:50,258 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:50,258 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:B 2024-11-12T06:49:50,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:49:50,258 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. because compaction request was cancelled 2024-11-12T06:49:50,258 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efaada0e985d01f31ce286661347290d:C 2024-11-12T06:49:50,258 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing efaada0e985d01f31ce286661347290d 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-12T06:49:50,258 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=A 2024-11-12T06:49:50,258 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:50,258 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=B 2024-11-12T06:49:50,258 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:50,258 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efaada0e985d01f31ce286661347290d, store=C 2024-11-12T06:49:50,258 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:50,261 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/84b5c8ffa81348fba4b40995682cb6e7 is 50, key is test_row_2/A:col10/1731394189007/Put/seqid=0 2024-11-12T06:49:50,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742331_1507 (size=7415) 2024-11-12T06:49:50,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T06:49:50,665 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=804 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/84b5c8ffa81348fba4b40995682cb6e7 2024-11-12T06:49:50,670 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/6cadc497e93a4fc186c28c146f050526 is 50, key is test_row_2/B:col10/1731394189007/Put/seqid=0 2024-11-12T06:49:50,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742332_1508 (size=7415) 2024-11-12T06:49:51,075 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=804 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/6cadc497e93a4fc186c28c146f050526 2024-11-12T06:49:51,087 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/249ef18bce4c419093339779d087757c is 50, key is test_row_2/C:col10/1731394189007/Put/seqid=0 2024-11-12T06:49:51,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742333_1509 (size=7415) 2024-11-12T06:49:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T06:49:51,492 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=804 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/249ef18bce4c419093339779d087757c 2024-11-12T06:49:51,504 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/A/84b5c8ffa81348fba4b40995682cb6e7 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/84b5c8ffa81348fba4b40995682cb6e7 2024-11-12T06:49:51,510 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/84b5c8ffa81348fba4b40995682cb6e7, entries=50, sequenceid=804, filesize=7.2 K 2024-11-12T06:49:51,511 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/B/6cadc497e93a4fc186c28c146f050526 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/6cadc497e93a4fc186c28c146f050526 2024-11-12T06:49:51,515 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/6cadc497e93a4fc186c28c146f050526, entries=50, sequenceid=804, filesize=7.2 K 2024-11-12T06:49:51,515 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/.tmp/C/249ef18bce4c419093339779d087757c as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/249ef18bce4c419093339779d087757c 2024-11-12T06:49:51,518 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/249ef18bce4c419093339779d087757c, entries=50, sequenceid=804, filesize=7.2 K 2024-11-12T06:49:51,519 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for efaada0e985d01f31ce286661347290d in 1261ms, sequenceid=804, compaction requested=true 2024-11-12T06:49:51,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/0f7a8b4b4902472b902c30638a711d6c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/91e216b1083649a69d81f5cde3d0bccd, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/fda3702b7b8f473fb2fde3b74b511271, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/afda0b19426f4a569776bbc573408c5e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/2a6db3a05f6c405e8eee3fcf1eb6cc9c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/69a229600e314b168e7359cc33dacd3f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/73d1460d4bda4f5cac4088e229ebd762, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/5af56074f6db4333a9a5e0e9df351c87, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/ad2c55db68224deea8722255c6df169b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/385770d83bf949c48cd9a9373635a135, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/b9c982a7d37446cc994aa11096a0e994, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/8863092ea68144cda37116a944949d3c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/5a26ee24add04aa09a0cea692e27b203, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/9b859f0941f942da8fedbf4d4935f585, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/ea0186fecdcc4324a612c6464531cda7, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/288f90c263f54f60b2e0f7ab2a4a671b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/06122280c929411ea60ff3758d83c401, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/41b13b651a4e4df5921b58b881aa8efe, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/5eadaaa85bc84355bb1c9188d99125a2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/e552594cc81b48efb410aafed15669da, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/a10325430d6e45d38cad3da7b2646942, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/9de5a04e8b4646e9a783539b28c7478b] to archive 2024-11-12T06:49:51,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T06:49:51,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/0f7a8b4b4902472b902c30638a711d6c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/0f7a8b4b4902472b902c30638a711d6c 2024-11-12T06:49:51,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/91e216b1083649a69d81f5cde3d0bccd to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/91e216b1083649a69d81f5cde3d0bccd 2024-11-12T06:49:51,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/fda3702b7b8f473fb2fde3b74b511271 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/fda3702b7b8f473fb2fde3b74b511271 2024-11-12T06:49:51,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/afda0b19426f4a569776bbc573408c5e to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/afda0b19426f4a569776bbc573408c5e 2024-11-12T06:49:51,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/2a6db3a05f6c405e8eee3fcf1eb6cc9c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/2a6db3a05f6c405e8eee3fcf1eb6cc9c 2024-11-12T06:49:51,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/69a229600e314b168e7359cc33dacd3f to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/69a229600e314b168e7359cc33dacd3f 2024-11-12T06:49:51,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/73d1460d4bda4f5cac4088e229ebd762 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/73d1460d4bda4f5cac4088e229ebd762 2024-11-12T06:49:51,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/5af56074f6db4333a9a5e0e9df351c87 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/5af56074f6db4333a9a5e0e9df351c87 2024-11-12T06:49:51,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/ad2c55db68224deea8722255c6df169b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/ad2c55db68224deea8722255c6df169b 2024-11-12T06:49:51,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/385770d83bf949c48cd9a9373635a135 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/385770d83bf949c48cd9a9373635a135 2024-11-12T06:49:51,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/b9c982a7d37446cc994aa11096a0e994 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/b9c982a7d37446cc994aa11096a0e994 2024-11-12T06:49:51,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/8863092ea68144cda37116a944949d3c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/8863092ea68144cda37116a944949d3c 2024-11-12T06:49:51,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/5a26ee24add04aa09a0cea692e27b203 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/5a26ee24add04aa09a0cea692e27b203 2024-11-12T06:49:51,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/9b859f0941f942da8fedbf4d4935f585 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/9b859f0941f942da8fedbf4d4935f585 2024-11-12T06:49:51,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/ea0186fecdcc4324a612c6464531cda7 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/ea0186fecdcc4324a612c6464531cda7 2024-11-12T06:49:51,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/288f90c263f54f60b2e0f7ab2a4a671b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/288f90c263f54f60b2e0f7ab2a4a671b 2024-11-12T06:49:51,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/06122280c929411ea60ff3758d83c401 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/06122280c929411ea60ff3758d83c401 2024-11-12T06:49:51,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/41b13b651a4e4df5921b58b881aa8efe to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/41b13b651a4e4df5921b58b881aa8efe 2024-11-12T06:49:51,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/5eadaaa85bc84355bb1c9188d99125a2 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/5eadaaa85bc84355bb1c9188d99125a2 2024-11-12T06:49:51,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/e552594cc81b48efb410aafed15669da to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/e552594cc81b48efb410aafed15669da 2024-11-12T06:49:51,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/a10325430d6e45d38cad3da7b2646942 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/a10325430d6e45d38cad3da7b2646942 2024-11-12T06:49:51,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/9de5a04e8b4646e9a783539b28c7478b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/9de5a04e8b4646e9a783539b28c7478b 2024-11-12T06:49:51,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/26bb813b231c4668be17ab531bc4007e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/06609cc4ea1d44029d0739a386aad6c2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/0149efffb6f84f8695ba9047e8935bac, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/7602a8f0aa0b4c699c61a29d212c4485, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/7455e0c16f734e3e8845bea3251689fa, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/9e5441dd350140d789238b817de4833f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/b84529bdee2c445785e856098731cd10, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/0b957fee8c0e44b09d7c1adb1100f54b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/26eb542f04f249f899c0b272fddbbcb9, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/ebdfdfc6e76e466295750d58d0535375, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/19e071138ef44d1b812623f74276165a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/5b91adae434745749b0c7f058635fbfd, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/fc32667035f34480bd5c39e62f2aa227, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/e3b45b147b124011b7ae509eb50014bb, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/22436698610f41f1b13f94ffee766446, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/a575a076060a4172937946fc5d91f201, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/9750181af3c54b35973d5852cebfef0c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/b4f753b064b1416d8ce6f9daf78ef5d9, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/20f5af66fcf449029d0a2c0232a320e1, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/cd71880a327f469a88501fc7a4be7e8c, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/1a6c0405ec5a4dbeb8ad6b5731e21705, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/90df3fed6dbf4b7ab06b354f5e74cb92, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/123d42f204f24ef896110221926f4430, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/eef7a77402ae45e88d5fbd3997509190, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/caa15e69b51e4300a9151d8a3fccc93c] to archive 2024-11-12T06:49:51,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T06:49:51,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/26bb813b231c4668be17ab531bc4007e to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/26bb813b231c4668be17ab531bc4007e 2024-11-12T06:49:51,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/06609cc4ea1d44029d0739a386aad6c2 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/06609cc4ea1d44029d0739a386aad6c2 2024-11-12T06:49:51,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/0149efffb6f84f8695ba9047e8935bac to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/0149efffb6f84f8695ba9047e8935bac 2024-11-12T06:49:51,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/7602a8f0aa0b4c699c61a29d212c4485 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/7602a8f0aa0b4c699c61a29d212c4485 2024-11-12T06:49:51,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/7455e0c16f734e3e8845bea3251689fa to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/7455e0c16f734e3e8845bea3251689fa 2024-11-12T06:49:51,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/9e5441dd350140d789238b817de4833f to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/9e5441dd350140d789238b817de4833f 2024-11-12T06:49:51,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/b84529bdee2c445785e856098731cd10 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/b84529bdee2c445785e856098731cd10 2024-11-12T06:49:51,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/0b957fee8c0e44b09d7c1adb1100f54b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/0b957fee8c0e44b09d7c1adb1100f54b 2024-11-12T06:49:51,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/26eb542f04f249f899c0b272fddbbcb9 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/26eb542f04f249f899c0b272fddbbcb9 2024-11-12T06:49:51,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/ebdfdfc6e76e466295750d58d0535375 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/ebdfdfc6e76e466295750d58d0535375 2024-11-12T06:49:51,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/19e071138ef44d1b812623f74276165a to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/19e071138ef44d1b812623f74276165a 2024-11-12T06:49:51,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/5b91adae434745749b0c7f058635fbfd to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/5b91adae434745749b0c7f058635fbfd 2024-11-12T06:49:51,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/fc32667035f34480bd5c39e62f2aa227 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/fc32667035f34480bd5c39e62f2aa227 2024-11-12T06:49:51,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/e3b45b147b124011b7ae509eb50014bb to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/e3b45b147b124011b7ae509eb50014bb 2024-11-12T06:49:51,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/22436698610f41f1b13f94ffee766446 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/22436698610f41f1b13f94ffee766446 2024-11-12T06:49:51,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/a575a076060a4172937946fc5d91f201 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/a575a076060a4172937946fc5d91f201 2024-11-12T06:49:51,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/9750181af3c54b35973d5852cebfef0c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/9750181af3c54b35973d5852cebfef0c 2024-11-12T06:49:51,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/b4f753b064b1416d8ce6f9daf78ef5d9 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/b4f753b064b1416d8ce6f9daf78ef5d9 2024-11-12T06:49:51,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/20f5af66fcf449029d0a2c0232a320e1 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/20f5af66fcf449029d0a2c0232a320e1 2024-11-12T06:49:51,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/cd71880a327f469a88501fc7a4be7e8c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/cd71880a327f469a88501fc7a4be7e8c 2024-11-12T06:49:51,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/1a6c0405ec5a4dbeb8ad6b5731e21705 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/1a6c0405ec5a4dbeb8ad6b5731e21705 2024-11-12T06:49:51,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/90df3fed6dbf4b7ab06b354f5e74cb92 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/90df3fed6dbf4b7ab06b354f5e74cb92 2024-11-12T06:49:51,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/123d42f204f24ef896110221926f4430 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/123d42f204f24ef896110221926f4430 2024-11-12T06:49:51,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/eef7a77402ae45e88d5fbd3997509190 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/eef7a77402ae45e88d5fbd3997509190 2024-11-12T06:49:51,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/caa15e69b51e4300a9151d8a3fccc93c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/caa15e69b51e4300a9151d8a3fccc93c 2024-11-12T06:49:51,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/e3750f06387442d1a4000dca780de1f1, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/06775c328594473bbb4287a64868bb81, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/59dacdbadb8e42f3be9a07debff7770f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/08f57a6a77924fcb82b40c8436fd1d56, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/7c9ef7f61c6f483aaf10e98be81b4316, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/338710321cd34d98b0ad92bbba717a52, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/c331efbd97854a62b5cd7563a95430de, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/566a652fa4bb4bcda0a7239771d7b481, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/88729fb5bc6c4ce88ae2984d005051fa, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/6d05d49a246b44428c1c824b7ad20608, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/cf2f1a0a53de48299bd099ebb9250bf8, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/c50f7212b5754fccafb141414cc86362, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/e547c2d018444e1cbcae5a28e4b9c5e3, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/f3634668dfd34c2aa8793f865d18d783, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/9a6392785bc648fa89f8269601fcb7a5, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/2adb27a099ec49b8b9908fba76fb39c7, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/22de97d685bd4bc9af6db03e93b36f35, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/a6834c4ec7664fdda77ce4f56f5abe88, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/7ba6e074210445cc977b10a5dfecc62b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/7c60b06dd04a433aa12ac88761eeacb1, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/c73622740bac43079c6ab1bb36757464, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/09077310b8bf4043aacd8ca122d0327f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/52ea2859fa0146da9d7a8533326faf56, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/087bfe85974b45be8841f41917f6c8c5, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/f548f455b1694f5cb011f05793aae072] to archive 2024-11-12T06:49:51,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T06:49:51,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/e3750f06387442d1a4000dca780de1f1 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/e3750f06387442d1a4000dca780de1f1 2024-11-12T06:49:51,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/06775c328594473bbb4287a64868bb81 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/06775c328594473bbb4287a64868bb81 2024-11-12T06:49:51,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/59dacdbadb8e42f3be9a07debff7770f to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/59dacdbadb8e42f3be9a07debff7770f 2024-11-12T06:49:51,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/08f57a6a77924fcb82b40c8436fd1d56 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/08f57a6a77924fcb82b40c8436fd1d56 2024-11-12T06:49:51,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/7c9ef7f61c6f483aaf10e98be81b4316 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/7c9ef7f61c6f483aaf10e98be81b4316 2024-11-12T06:49:51,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/338710321cd34d98b0ad92bbba717a52 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/338710321cd34d98b0ad92bbba717a52 2024-11-12T06:49:51,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/c331efbd97854a62b5cd7563a95430de to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/c331efbd97854a62b5cd7563a95430de 2024-11-12T06:49:51,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/566a652fa4bb4bcda0a7239771d7b481 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/566a652fa4bb4bcda0a7239771d7b481 2024-11-12T06:49:51,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/88729fb5bc6c4ce88ae2984d005051fa to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/88729fb5bc6c4ce88ae2984d005051fa 2024-11-12T06:49:51,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/6d05d49a246b44428c1c824b7ad20608 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/6d05d49a246b44428c1c824b7ad20608 2024-11-12T06:49:51,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/cf2f1a0a53de48299bd099ebb9250bf8 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/cf2f1a0a53de48299bd099ebb9250bf8 2024-11-12T06:49:51,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/c50f7212b5754fccafb141414cc86362 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/c50f7212b5754fccafb141414cc86362 2024-11-12T06:49:51,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/e547c2d018444e1cbcae5a28e4b9c5e3 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/e547c2d018444e1cbcae5a28e4b9c5e3 2024-11-12T06:49:51,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/f3634668dfd34c2aa8793f865d18d783 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/f3634668dfd34c2aa8793f865d18d783 2024-11-12T06:49:51,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/9a6392785bc648fa89f8269601fcb7a5 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/9a6392785bc648fa89f8269601fcb7a5 2024-11-12T06:49:51,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/2adb27a099ec49b8b9908fba76fb39c7 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/2adb27a099ec49b8b9908fba76fb39c7 2024-11-12T06:49:51,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/22de97d685bd4bc9af6db03e93b36f35 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/22de97d685bd4bc9af6db03e93b36f35 2024-11-12T06:49:51,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/a6834c4ec7664fdda77ce4f56f5abe88 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/a6834c4ec7664fdda77ce4f56f5abe88 2024-11-12T06:49:51,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/7ba6e074210445cc977b10a5dfecc62b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/7ba6e074210445cc977b10a5dfecc62b 2024-11-12T06:49:51,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/7c60b06dd04a433aa12ac88761eeacb1 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/7c60b06dd04a433aa12ac88761eeacb1 2024-11-12T06:49:51,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/c73622740bac43079c6ab1bb36757464 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/c73622740bac43079c6ab1bb36757464 2024-11-12T06:49:51,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/09077310b8bf4043aacd8ca122d0327f to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/09077310b8bf4043aacd8ca122d0327f 2024-11-12T06:49:51,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/52ea2859fa0146da9d7a8533326faf56 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/52ea2859fa0146da9d7a8533326faf56 2024-11-12T06:49:51,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/087bfe85974b45be8841f41917f6c8c5 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/087bfe85974b45be8841f41917f6c8c5 2024-11-12T06:49:51,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/f548f455b1694f5cb011f05793aae072 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/f548f455b1694f5cb011f05793aae072 2024-11-12T06:49:51,589 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/recovered.edits/807.seqid, newMaxSeqId=807, maxSeqId=1 2024-11-12T06:49:51,590 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. 2024-11-12T06:49:51,590 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for efaada0e985d01f31ce286661347290d: Waiting for close lock at 1731394189388Running coprocessor pre-close hooks at 1731394189388Disabling compacts and flushes for region at 1731394189388Disabling writes for close at 1731394190258 (+870 ms)Obtaining lock to block concurrent updates at 1731394190258Preparing flush snapshotting stores in efaada0e985d01f31ce286661347290d at 1731394190258Finished memstore snapshotting TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731394190258Flushing stores of TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d. at 1731394190258Flushing efaada0e985d01f31ce286661347290d/A: creating writer at 1731394190258Flushing efaada0e985d01f31ce286661347290d/A: appending metadata at 1731394190261 (+3 ms)Flushing efaada0e985d01f31ce286661347290d/A: closing flushed file at 1731394190261Flushing efaada0e985d01f31ce286661347290d/B: creating writer at 1731394190668 (+407 ms)Flushing efaada0e985d01f31ce286661347290d/B: appending metadata at 1731394190670 (+2 ms)Flushing efaada0e985d01f31ce286661347290d/B: closing flushed file at 1731394190670Flushing efaada0e985d01f31ce286661347290d/C: creating writer at 1731394191084 (+414 ms)Flushing efaada0e985d01f31ce286661347290d/C: appending metadata at 1731394191086 (+2 ms)Flushing efaada0e985d01f31ce286661347290d/C: closing flushed file at 1731394191086Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@266a2477: reopening flushed file at 1731394191502 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6062c87b: reopening flushed file at 1731394191510 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@522c8a02: reopening flushed file at 1731394191515 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for efaada0e985d01f31ce286661347290d in 1261ms, sequenceid=804, compaction requested=true at 1731394191519 (+4 ms)Writing region close event to WAL at 1731394191585 (+66 ms)Running coprocessor post-close hooks at 1731394191590 (+5 ms)Closed at 1731394191590 2024-11-12T06:49:51,593 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed efaada0e985d01f31ce286661347290d 2024-11-12T06:49:51,593 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=efaada0e985d01f31ce286661347290d, regionState=CLOSED 2024-11-12T06:49:51,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 because future has completed 2024-11-12T06:49:51,597 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-12T06:49:51,598 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure efaada0e985d01f31ce286661347290d, server=9f0beee3d6b6,38777,1731394047622 in 2.3610 sec 2024-11-12T06:49:51,599 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-12T06:49:51,599 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efaada0e985d01f31ce286661347290d, UNASSIGN in 2.3660 sec 2024-11-12T06:49:51,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-12T06:49:51,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3690 sec 2024-11-12T06:49:51,604 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394191603"}]},"ts":"1731394191603"} 2024-11-12T06:49:51,605 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T06:49:51,605 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T06:49:51,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.3840 sec 2024-11-12T06:49:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T06:49:53,397 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:49:53,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T06:49:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:49:53,399 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:49:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-12T06:49:53,400 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:49:53,402 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d 2024-11-12T06:49:53,404 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A, FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B, FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C, FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/recovered.edits] 2024-11-12T06:49:53,406 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/3a41506ba08745c59c2f1a0eb345c6ce to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/3a41506ba08745c59c2f1a0eb345c6ce 2024-11-12T06:49:53,407 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/84b5c8ffa81348fba4b40995682cb6e7 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/84b5c8ffa81348fba4b40995682cb6e7 2024-11-12T06:49:53,408 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/b7a7e6a5a60445d783abbffec9ed1bff to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/b7a7e6a5a60445d783abbffec9ed1bff 2024-11-12T06:49:53,409 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/cf836aeb8c2d4149a42d3820eee5ec1e to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/A/cf836aeb8c2d4149a42d3820eee5ec1e 2024-11-12T06:49:53,410 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/188b3d1d961c4ecc8248740743342395 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/188b3d1d961c4ecc8248740743342395 2024-11-12T06:49:53,411 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/6cadc497e93a4fc186c28c146f050526 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/6cadc497e93a4fc186c28c146f050526 2024-11-12T06:49:53,411 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/da12976ea3504696bf102a4db0f66c20 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/da12976ea3504696bf102a4db0f66c20 2024-11-12T06:49:53,412 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/f2359636fb2c4b719ca2c3a33e1a7b39 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/B/f2359636fb2c4b719ca2c3a33e1a7b39 2024-11-12T06:49:53,413 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/249ef18bce4c419093339779d087757c to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/249ef18bce4c419093339779d087757c 2024-11-12T06:49:53,414 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/523250a62d0a4fe6834071cf24c6ee79 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/523250a62d0a4fe6834071cf24c6ee79 2024-11-12T06:49:53,415 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/5e75327637f54bb8a2099fdc10c2fc75 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/5e75327637f54bb8a2099fdc10c2fc75 2024-11-12T06:49:53,415 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/66ef588f25444303b868a7868757029d to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/C/66ef588f25444303b868a7868757029d 2024-11-12T06:49:53,417 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/recovered.edits/807.seqid to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d/recovered.edits/807.seqid 2024-11-12T06:49:53,417 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/efaada0e985d01f31ce286661347290d 2024-11-12T06:49:53,417 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T06:49:53,419 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:49:53,420 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T06:49:53,421 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T06:49:53,422 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:49:53,422 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T06:49:53,422 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731394193422"}]},"ts":"9223372036854775807"} 2024-11-12T06:49:53,424 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T06:49:53,424 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => efaada0e985d01f31ce286661347290d, NAME => 'TestAcidGuarantees,,1731394164919.efaada0e985d01f31ce286661347290d.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T06:49:53,424 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T06:49:53,424 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731394193424"}]},"ts":"9223372036854775807"} 2024-11-12T06:49:53,425 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T06:49:53,426 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:49:53,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 28 msec 2024-11-12T06:49:53,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-12T06:49:53,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T06:49:53,507 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:49:53,516 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 248) - Thread LEAK? -, OpenFileDescriptor=447 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=310 (was 333), ProcessCount=11 (was 11), AvailableMemoryMB=6485 (was 6567) 2024-11-12T06:49:53,523 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=310, ProcessCount=11, AvailableMemoryMB=6485 2024-11-12T06:49:53,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T06:49:53,524 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T06:49:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T06:49:53,526 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T06:49:53,526 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:53,526 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-11-12T06:49:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-12T06:49:53,526 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T06:49:53,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742334_1510 (size=1061) 2024-11-12T06:49:53,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-12T06:49:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-12T06:49:53,934 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e 2024-11-12T06:49:53,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742335_1511 (size=53) 2024-11-12T06:49:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-12T06:49:54,340 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T06:49:54,340 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f8bcab44974ca57906f4d57bd4289b93, disabling compactions & flushes 2024-11-12T06:49:54,340 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:54,340 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:54,340 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. after waiting 0 ms 2024-11-12T06:49:54,340 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:54,340 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:54,340 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f8bcab44974ca57906f4d57bd4289b93: Waiting for close lock at 1731394194340Disabling compacts and flushes for region at 1731394194340Disabling writes for close at 1731394194340Writing region close event to WAL at 1731394194340Closed at 1731394194340 2024-11-12T06:49:54,341 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T06:49:54,341 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731394194341"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731394194341"}]},"ts":"1731394194341"} 2024-11-12T06:49:54,342 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T06:49:54,343 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T06:49:54,343 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394194343"}]},"ts":"1731394194343"} 2024-11-12T06:49:54,344 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T06:49:54,344 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8bcab44974ca57906f4d57bd4289b93, ASSIGN}] 2024-11-12T06:49:54,345 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8bcab44974ca57906f4d57bd4289b93, ASSIGN 2024-11-12T06:49:54,346 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8bcab44974ca57906f4d57bd4289b93, ASSIGN; state=OFFLINE, location=9f0beee3d6b6,38777,1731394047622; forceNewPlan=false, retain=false 2024-11-12T06:49:54,496 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=f8bcab44974ca57906f4d57bd4289b93, regionState=OPENING, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:54,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8bcab44974ca57906f4d57bd4289b93, ASSIGN because future has completed 2024-11-12T06:49:54,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622}] 2024-11-12T06:49:54,652 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:54,652 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} 2024-11-12T06:49:54,652 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:54,652 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T06:49:54,652 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:54,652 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:54,653 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:54,654 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:49:54,654 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8bcab44974ca57906f4d57bd4289b93 columnFamilyName A 2024-11-12T06:49:54,654 DEBUG [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:54,655 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] regionserver.HStore(327): Store=f8bcab44974ca57906f4d57bd4289b93/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:49:54,655 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:54,656 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:49:54,656 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8bcab44974ca57906f4d57bd4289b93 columnFamilyName B 2024-11-12T06:49:54,656 DEBUG [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:54,656 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] regionserver.HStore(327): Store=f8bcab44974ca57906f4d57bd4289b93/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:49:54,656 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:54,657 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:49:54,657 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8bcab44974ca57906f4d57bd4289b93 columnFamilyName C 2024-11-12T06:49:54,657 DEBUG [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:54,657 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] regionserver.HStore(327): Store=f8bcab44974ca57906f4d57bd4289b93/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:49:54,658 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:54,658 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:54,658 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:54,658 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:54,659 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:54,659 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:54,660 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T06:49:54,661 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:54,662 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T06:49:54,662 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened f8bcab44974ca57906f4d57bd4289b93; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62437480, jitterRate=-0.06960904598236084}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T06:49:54,662 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:54,663 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for f8bcab44974ca57906f4d57bd4289b93: Running coprocessor pre-open hook at 1731394194652Writing region info on filesystem at 1731394194652Initializing all the Stores at 1731394194653 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394194653Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394194653Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394194653Cleaning up temporary data from old regions at 1731394194659 (+6 ms)Running coprocessor post-open hooks at 1731394194662 (+3 ms)Region opened successfully at 1731394194663 (+1 ms) 2024-11-12T06:49:54,663 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., pid=140, masterSystemTime=1731394194649 2024-11-12T06:49:54,665 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:54,665 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:54,665 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=f8bcab44974ca57906f4d57bd4289b93, regionState=OPEN, openSeqNum=2, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:54,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 because future has completed 2024-11-12T06:49:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-12T06:49:54,668 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-12T06:49:54,668 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 in 169 msec 2024-11-12T06:49:54,669 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-12T06:49:54,669 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8bcab44974ca57906f4d57bd4289b93, ASSIGN in 324 msec 2024-11-12T06:49:54,670 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T06:49:54,670 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394194670"}]},"ts":"1731394194670"} 2024-11-12T06:49:54,671 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T06:49:54,671 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T06:49:54,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2024-11-12T06:49:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-12T06:49:55,678 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:49:55,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1832522d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:55,679 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:49:55,679 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:49:55,679 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:49:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:49:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:49:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3117b36c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:49:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:49:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:55,681 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:49:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6560b57e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:55,681 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:49:55,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:49:55,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:49:55,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:49:55,684 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T06:49:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 9f0beee3d6b6,43031,1731394046864 2024-11-12T06:49:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3070d784 2024-11-12T06:49:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T06:49:55,685 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T06:49:55,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T06:49:55,686 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T06:49:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-12T06:49:55,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742336_1512 (size=1097) 2024-11-12T06:49:55,970 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T06:49:56,097 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-12T06:49:56,097 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-12T06:49:56,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T06:49:56,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8bcab44974ca57906f4d57bd4289b93, REOPEN/MOVE}] 2024-11-12T06:49:56,102 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8bcab44974ca57906f4d57bd4289b93, REOPEN/MOVE 2024-11-12T06:49:56,103 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=f8bcab44974ca57906f4d57bd4289b93, regionState=CLOSING, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:56,103 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8bcab44974ca57906f4d57bd4289b93, REOPEN/MOVE because future has completed 2024-11-12T06:49:56,104 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T06:49:56,104 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622}] 2024-11-12T06:49:56,254 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:56,254 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T06:49:56,254 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing f8bcab44974ca57906f4d57bd4289b93, disabling compactions & flushes 2024-11-12T06:49:56,255 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:56,255 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:56,255 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. after waiting 0 ms 2024-11-12T06:49:56,255 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:56,258 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T06:49:56,258 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:56,258 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for f8bcab44974ca57906f4d57bd4289b93: Waiting for close lock at 1731394196254Running coprocessor pre-close hooks at 1731394196254Disabling compacts and flushes for region at 1731394196254Disabling writes for close at 1731394196255 (+1 ms)Writing region close event to WAL at 1731394196255Running coprocessor post-close hooks at 1731394196258 (+3 ms)Closed at 1731394196258 2024-11-12T06:49:56,258 WARN [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: f8bcab44974ca57906f4d57bd4289b93 to self. 2024-11-12T06:49:56,260 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:56,260 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=f8bcab44974ca57906f4d57bd4289b93, regionState=CLOSED 2024-11-12T06:49:56,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 because future has completed 2024-11-12T06:49:56,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-12T06:49:56,264 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 in 158 msec 2024-11-12T06:49:56,264 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8bcab44974ca57906f4d57bd4289b93, REOPEN/MOVE; state=CLOSED, location=9f0beee3d6b6,38777,1731394047622; forceNewPlan=false, retain=true 2024-11-12T06:49:56,415 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=f8bcab44974ca57906f4d57bd4289b93, regionState=OPENING, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:56,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8bcab44974ca57906f4d57bd4289b93, REOPEN/MOVE because future has completed 2024-11-12T06:49:56,416 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622}] 2024-11-12T06:49:56,570 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:56,570 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} 2024-11-12T06:49:56,570 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:56,570 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T06:49:56,570 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:56,570 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:56,571 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:56,572 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:49:56,572 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8bcab44974ca57906f4d57bd4289b93 columnFamilyName A 2024-11-12T06:49:56,573 DEBUG [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:56,573 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] regionserver.HStore(327): Store=f8bcab44974ca57906f4d57bd4289b93/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:49:56,573 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:56,574 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:49:56,574 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8bcab44974ca57906f4d57bd4289b93 columnFamilyName B 2024-11-12T06:49:56,574 DEBUG [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:56,574 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] regionserver.HStore(327): Store=f8bcab44974ca57906f4d57bd4289b93/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:49:56,574 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:56,575 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T06:49:56,575 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8bcab44974ca57906f4d57bd4289b93 columnFamilyName C 2024-11-12T06:49:56,575 DEBUG [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:56,575 INFO [StoreOpener-f8bcab44974ca57906f4d57bd4289b93-1 {}] regionserver.HStore(327): Store=f8bcab44974ca57906f4d57bd4289b93/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T06:49:56,575 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:56,575 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:56,576 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:56,577 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:56,577 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:56,577 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:56,578 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T06:49:56,579 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:56,579 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened f8bcab44974ca57906f4d57bd4289b93; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75414535, jitterRate=0.12376414239406586}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T06:49:56,579 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:56,580 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for f8bcab44974ca57906f4d57bd4289b93: Running coprocessor pre-open hook at 1731394196570Writing region info on filesystem at 1731394196570Initializing all the Stores at 1731394196571 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394196571Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394196571Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731394196571Cleaning up temporary data from old regions at 1731394196577 (+6 ms)Running coprocessor post-open hooks at 1731394196579 (+2 ms)Region opened successfully at 1731394196580 (+1 ms) 2024-11-12T06:49:56,580 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., pid=145, masterSystemTime=1731394196568 2024-11-12T06:49:56,582 DEBUG [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:56,582 INFO [RS_OPEN_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:56,582 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=f8bcab44974ca57906f4d57bd4289b93, regionState=OPEN, openSeqNum=5, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:56,583 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 because future has completed 2024-11-12T06:49:56,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-11-12T06:49:56,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 in 168 msec 2024-11-12T06:49:56,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-12T06:49:56,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8bcab44974ca57906f4d57bd4289b93, REOPEN/MOVE in 484 msec 2024-11-12T06:49:56,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-12T06:49:56,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 488 msec 2024-11-12T06:49:56,589 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 901 msec 2024-11-12T06:49:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-12T06:49:56,590 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:49:56,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22faf5ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,591 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:49:56,591 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:49:56,591 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:49:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:49:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:49:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f84566, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:49:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:49:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:56,593 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:49:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7549d38a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,594 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:49:56,594 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:49:56,594 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:49:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:49:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:49:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27e79f85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:49:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:49:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:56,596 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:49:56,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54155653, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,597 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:49:56,597 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:49:56,597 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:49:56,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:49:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:49:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aafeeb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:49:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:49:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:56,598 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:49:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e2ed89a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,600 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:49:56,600 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:49:56,601 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:49:56,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:49:56,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:49:56,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a806057, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:49:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:49:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:56,602 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:49:56,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24348d6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,603 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:49:56,603 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:49:56,604 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:49:56,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:49:56,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:49:56,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@730f9c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:49:56,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:49:56,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:56,605 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48876, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:49:56,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e281519, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,606 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:49:56,607 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:49:56,607 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:49:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:49:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:49:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33b8decf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:49:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:49:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:56,608 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:49:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f6fef98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,609 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:49:56,610 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:49:56,610 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:49:56,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:49:56,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:49:56,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5bb66f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:49:56,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:49:56,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:56,612 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48914, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:49:56,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@646855ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,613 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:49:56,613 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:49:56,614 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:49:56,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:49:56,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:49:56,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cbf6518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:49:56,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:49:56,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:56,615 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:49:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74513805, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,617 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:49:56,617 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:49:56,617 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:49:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:49:56,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:49:56,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@523061ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:49:56,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:49:56,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:56,619 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:49:56,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1db0e35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,619 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f0beee3d6b6,43031,-1 for getting cluster id 2024-11-12T06:49:56,620 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T06:49:56,620 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6fe3efea-1c5d-4422-afa1-c9a075a9eabd' 2024-11-12T06:49:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T06:49:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6fe3efea-1c5d-4422-afa1-c9a075a9eabd" 2024-11-12T06:49:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bf33eac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f0beee3d6b6,43031,-1] 2024-11-12T06:49:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T06:49:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:49:56,621 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T06:49:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5553332c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T06:49:56,622 DEBUG [Thread-2292 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:49:56,622 DEBUG [Thread-2309 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:49:56,623 DEBUG [Thread-2296 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:49:56,623 DEBUG [Thread-2307 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:49:56,623 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:49:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:49:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:49:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:49:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:49:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:49:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:49:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-12T06:49:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:49:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:49:56,625 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:49:56,625 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:49:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T06:49:56,625 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:49:56,625 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:49:56,625 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:49:56,625 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:49:56,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:49:56,627 DEBUG [Thread-2290 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:49:56,627 DEBUG [Thread-2301 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:49:56,627 DEBUG [Thread-2298 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:49:56,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:49:56,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:49:56,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:49:56,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:49:56,629 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:49:56,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:49:56,630 DEBUG [Thread-2303 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:49:56,631 DEBUG [Thread-2305 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:49:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:49:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:49:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:49:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:49:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:49:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:49:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:49:56,632 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:49:56,633 DEBUG [Thread-2294 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T06:49:56,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:49:56,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:49:56,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:49:56,634 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:49:56,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f0beee3d6b6,38777,1731394047622, seqNum=-1] 2024-11-12T06:49:56,634 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:49:56,634 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:49:56,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T06:49:56,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:49:56,637 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T06:49:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:56,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:49:56,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8bcab44974ca57906f4d57bd4289b93 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T06:49:56,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=A 2024-11-12T06:49:56,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:56,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=B 2024-11-12T06:49:56,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:56,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=C 2024-11-12T06:49:56,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:49:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:49:56,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5] 2024-11-12T06:49:56,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731394196645, max timestamp=1731394196661Num uniques 859; 2024-11-12T06:49:56,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T06:49:56,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394196648, max timestamp=1731394196662Num uniques 859; 2024-11-12T06:49:56,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394196648, max timestamp=1731394196662Num uniques 859; 2024-11-12T06:49:56,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:49:56,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:49:56,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111262527b28a6db4070899e8bd6554f49d9_f8bcab44974ca57906f4d57bd4289b93 is 50, key is test_row_0/A:col10/1731394196635/Put/seqid=0 2024-11-12T06:49:56,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742337_1513 (size=12154) 2024-11-12T06:49:56,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394256679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:56,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:56,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:56,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:56,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394256680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:56,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:56,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:56,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:56,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731394256681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:56,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:56,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:56,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:56,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394256682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:56,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:56,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:56,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:56,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394256684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T06:49:56,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T06:49:56,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:56,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:49:56,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:56,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:56,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:56,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:56,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394256787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:56,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394256787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:56,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731394256797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:56,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394256797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:56,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394256797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:56,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T06:49:56,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:56,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:49:56,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:56,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:56,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T06:49:57,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394256997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:57,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:57,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394256997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:57,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:57,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:57,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731394257007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:57,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:57,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394257008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:57,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:57,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394257008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:57,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:57,087 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:57,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T06:49:57,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:57,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:49:57,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:57,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:57,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:57,091 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111262527b28a6db4070899e8bd6554f49d9_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111262527b28a6db4070899e8bd6554f49d9_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:57,092 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/27e766350b6b476686bf0d0c00509ad7, store: [table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:49:57,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/27e766350b6b476686bf0d0c00509ad7 is 175, key is test_row_0/A:col10/1731394196635/Put/seqid=0 2024-11-12T06:49:57,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742338_1514 (size=30955) 2024-11-12T06:49:57,111 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T06:49:57,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T06:49:57,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:57,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:49:57,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:57,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:57,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T06:49:57,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394257307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:57,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394257307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:57,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731394257317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:57,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394257317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:57,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:57,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:57,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394257317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:57,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T06:49:57,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:57,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:49:57,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:57,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:57,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:57,498 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/27e766350b6b476686bf0d0c00509ad7 2024-11-12T06:49:57,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/eeae36d8b8004cc5b982d313da241cd0 is 50, key is test_row_0/B:col10/1731394196635/Put/seqid=0 2024-11-12T06:49:57,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742339_1515 (size=12001) 2024-11-12T06:49:57,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/eeae36d8b8004cc5b982d313da241cd0 2024-11-12T06:49:57,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T06:49:57,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:57,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:49:57,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:57,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:57,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:57,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/9168c3718c5244ca873721b493286347 is 50, key is test_row_0/C:col10/1731394196635/Put/seqid=0 2024-11-12T06:49:57,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742340_1516 (size=12001) 2024-11-12T06:49:57,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T06:49:57,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:57,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:49:57,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:57,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:57,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:57,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:57,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T06:49:57,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394257817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:57,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394257817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:57,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:57,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731394257827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:57,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:57,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394257827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:57,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:57,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:57,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:57,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394257828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:57,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:57,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:57,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T06:49:57,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:57,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:49:57,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:57,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:57,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:57,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:49:57,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/9168c3718c5244ca873721b493286347 2024-11-12T06:49:57,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/27e766350b6b476686bf0d0c00509ad7 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/27e766350b6b476686bf0d0c00509ad7 2024-11-12T06:49:57,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/27e766350b6b476686bf0d0c00509ad7, entries=150, sequenceid=14, filesize=30.2 K 2024-11-12T06:49:57,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/eeae36d8b8004cc5b982d313da241cd0 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/eeae36d8b8004cc5b982d313da241cd0 2024-11-12T06:49:57,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/eeae36d8b8004cc5b982d313da241cd0, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T06:49:57,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/9168c3718c5244ca873721b493286347 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/9168c3718c5244ca873721b493286347 2024-11-12T06:49:57,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/9168c3718c5244ca873721b493286347, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T06:49:57,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f8bcab44974ca57906f4d57bd4289b93 in 1334ms, sequenceid=14, compaction requested=false 2024-11-12T06:49:57,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:49:58,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T06:49:58,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:58,004 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing f8bcab44974ca57906f4d57bd4289b93 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-12T06:49:58,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=A 2024-11-12T06:49:58,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:58,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=B 2024-11-12T06:49:58,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:58,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=C 2024-11-12T06:49:58,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:49:58,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121917dc8aaecf40a8bbb5abba5d219538_f8bcab44974ca57906f4d57bd4289b93 is 50, key is test_row_0/A:col10/1731394196680/Put/seqid=0 2024-11-12T06:49:58,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742341_1517 (size=12154) 2024-11-12T06:49:58,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:58,438 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121917dc8aaecf40a8bbb5abba5d219538_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121917dc8aaecf40a8bbb5abba5d219538_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:58,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/ecf247e30d634231a659dbc76031f400, store: [table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:49:58,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/ecf247e30d634231a659dbc76031f400 is 175, key is test_row_0/A:col10/1731394196680/Put/seqid=0 2024-11-12T06:49:58,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742342_1518 (size=30955) 2024-11-12T06:49:58,456 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/ecf247e30d634231a659dbc76031f400 2024-11-12T06:49:58,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/b67ed65c42f9434084cfea89e1018be3 is 50, key is test_row_0/B:col10/1731394196680/Put/seqid=0 2024-11-12T06:49:58,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742343_1519 (size=12001) 2024-11-12T06:49:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T06:49:58,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:49:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:58,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394258838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:58,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394258841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:58,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:58,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:58,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:58,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394258842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:58,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:58,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:58,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:58,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394258842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:58,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:58,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:58,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:58,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731394258847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:58,849 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_0' in f8bcab44974ca57906f4d57bd4289b93 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:49:58,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:58,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:58,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:58,880 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/b67ed65c42f9434084cfea89e1018be3 2024-11-12T06:49:58,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/f12ff1de4deb4e988b0a633965dec541 is 50, key is test_row_0/C:col10/1731394196680/Put/seqid=0 2024-11-12T06:49:58,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742344_1520 (size=12001) 2024-11-12T06:49:58,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394258947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:58,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:58,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394258957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:58,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:58,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394258958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:58,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:58,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394258958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:59,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:59,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394259157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:59,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:59,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:59,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394259167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:59,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:59,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:59,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394259167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:59,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:59,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:59,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394259167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:59,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:59,294 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/f12ff1de4deb4e988b0a633965dec541 2024-11-12T06:49:59,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/ecf247e30d634231a659dbc76031f400 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/ecf247e30d634231a659dbc76031f400 2024-11-12T06:49:59,301 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/ecf247e30d634231a659dbc76031f400, entries=150, sequenceid=50, filesize=30.2 K 2024-11-12T06:49:59,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/b67ed65c42f9434084cfea89e1018be3 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/b67ed65c42f9434084cfea89e1018be3 2024-11-12T06:49:59,305 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/b67ed65c42f9434084cfea89e1018be3, entries=150, sequenceid=50, filesize=11.7 K 2024-11-12T06:49:59,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/f12ff1de4deb4e988b0a633965dec541 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/f12ff1de4deb4e988b0a633965dec541 2024-11-12T06:49:59,309 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/f12ff1de4deb4e988b0a633965dec541, entries=150, sequenceid=50, filesize=11.7 K 2024-11-12T06:49:59,310 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for f8bcab44974ca57906f4d57bd4289b93 in 1306ms, sequenceid=50, compaction requested=false 2024-11-12T06:49:59,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:49:59,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:49:59,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-12T06:49:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-12T06:49:59,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-12T06:49:59,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6860 sec 2024-11-12T06:49:59,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.6890 sec 2024-11-12T06:49:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:59,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8bcab44974ca57906f4d57bd4289b93 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T06:49:59,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=A 2024-11-12T06:49:59,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:59,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=B 2024-11-12T06:49:59,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:59,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=C 2024-11-12T06:49:59,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:49:59,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122e0670529b5f4d74bf777ecfe61f4b37_f8bcab44974ca57906f4d57bd4289b93 is 50, key is test_row_0/A:col10/1731394199468/Put/seqid=0 2024-11-12T06:49:59,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742345_1521 (size=12154) 2024-11-12T06:49:59,481 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:49:59,485 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122e0670529b5f4d74bf777ecfe61f4b37_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122e0670529b5f4d74bf777ecfe61f4b37_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:49:59,486 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/94560b9f89654d2baa74df72fca93d1b, store: [table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:49:59,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/94560b9f89654d2baa74df72fca93d1b is 175, key is test_row_0/A:col10/1731394199468/Put/seqid=0 2024-11-12T06:49:59,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742346_1522 (size=30955) 2024-11-12T06:49:59,497 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/94560b9f89654d2baa74df72fca93d1b 2024-11-12T06:49:59,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/165fd00313904924a9c8c00adcecc3d8 is 50, key is test_row_0/B:col10/1731394199468/Put/seqid=0 2024-11-12T06:49:59,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394259505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:59,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:59,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394259506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:59,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:59,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:59,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394199496, max timestamp=1731394199509Num uniques 859; 2024-11-12T06:49:59,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:59,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394199496, max timestamp=1731394199509Num uniques 859; 2024-11-12T06:49:59,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:49:59,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731394199496, max timestamp=1731394199509Num uniques 859; 2024-11-12T06:49:59,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-12T06:49:59,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742347_1523 (size=12001) 2024-11-12T06:49:59,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394259546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:59,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:59,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394259553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:59,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:59,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:59,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394259617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:59,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394259617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:59,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:59,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394259657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:59,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:59,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:59,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394259667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:59,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:59,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394259827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:59,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394259827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:59,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394259866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:59,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:59,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:49:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394259877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:49:59,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:49:59,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:49:59,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/165fd00313904924a9c8c00adcecc3d8 2024-11-12T06:49:59,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/e93a1f87da0e4ba1aed06eb3a93d2335 is 50, key is test_row_0/C:col10/1731394199468/Put/seqid=0 2024-11-12T06:49:59,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742348_1524 (size=12001) 2024-11-12T06:50:00,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:00,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394260137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:00,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:00,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394260137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:00,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:00,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:00,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394260177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:00,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394260187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:00,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:00,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/e93a1f87da0e4ba1aed06eb3a93d2335 2024-11-12T06:50:00,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/94560b9f89654d2baa74df72fca93d1b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/94560b9f89654d2baa74df72fca93d1b 2024-11-12T06:50:00,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/94560b9f89654d2baa74df72fca93d1b, entries=150, sequenceid=61, filesize=30.2 K 2024-11-12T06:50:00,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/165fd00313904924a9c8c00adcecc3d8 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/165fd00313904924a9c8c00adcecc3d8 2024-11-12T06:50:00,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/165fd00313904924a9c8c00adcecc3d8, entries=150, sequenceid=61, filesize=11.7 K 2024-11-12T06:50:00,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/e93a1f87da0e4ba1aed06eb3a93d2335 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/e93a1f87da0e4ba1aed06eb3a93d2335 2024-11-12T06:50:00,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/e93a1f87da0e4ba1aed06eb3a93d2335, entries=150, sequenceid=61, filesize=11.7 K 2024-11-12T06:50:00,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=124.84 KB/127835 for f8bcab44974ca57906f4d57bd4289b93 in 874ms, sequenceid=61, compaction requested=true 2024-11-12T06:50:00,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:00,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8bcab44974ca57906f4d57bd4289b93:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:50:00,343 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:50:00,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:00,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8bcab44974ca57906f4d57bd4289b93:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:50:00,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:00,343 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:50:00,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8bcab44974ca57906f4d57bd4289b93:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:50:00,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:50:00,344 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:50:00,344 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): f8bcab44974ca57906f4d57bd4289b93/A is initiating minor compaction (all files) 2024-11-12T06:50:00,344 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8bcab44974ca57906f4d57bd4289b93/A in TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:00,345 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/27e766350b6b476686bf0d0c00509ad7, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/ecf247e30d634231a659dbc76031f400, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/94560b9f89654d2baa74df72fca93d1b] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp, totalSize=90.7 K 2024-11-12T06:50:00,345 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:00,345 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. files: [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/27e766350b6b476686bf0d0c00509ad7, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/ecf247e30d634231a659dbc76031f400, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/94560b9f89654d2baa74df72fca93d1b] 2024-11-12T06:50:00,345 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:50:00,345 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): f8bcab44974ca57906f4d57bd4289b93/B is initiating minor compaction (all files) 2024-11-12T06:50:00,345 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8bcab44974ca57906f4d57bd4289b93/B in TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:00,345 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/eeae36d8b8004cc5b982d313da241cd0, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/b67ed65c42f9434084cfea89e1018be3, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/165fd00313904924a9c8c00adcecc3d8] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp, totalSize=35.2 K 2024-11-12T06:50:00,345 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting eeae36d8b8004cc5b982d313da241cd0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731394196634 2024-11-12T06:50:00,346 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27e766350b6b476686bf0d0c00509ad7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731394196634 2024-11-12T06:50:00,346 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting b67ed65c42f9434084cfea89e1018be3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731394196680 2024-11-12T06:50:00,346 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecf247e30d634231a659dbc76031f400, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731394196680 2024-11-12T06:50:00,346 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 165fd00313904924a9c8c00adcecc3d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731394198840 2024-11-12T06:50:00,346 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94560b9f89654d2baa74df72fca93d1b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731394198840 2024-11-12T06:50:00,357 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:00,358 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8bcab44974ca57906f4d57bd4289b93#B#compaction#441 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:50:00,358 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/679ea1a922cd457fb7904fb3b8bf1b29 is 50, key is test_row_0/B:col10/1731394199468/Put/seqid=0 2024-11-12T06:50:00,359 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112c1ac58eccf4444ec9be71f8e4df9d0bb_f8bcab44974ca57906f4d57bd4289b93 store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:00,362 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112c1ac58eccf4444ec9be71f8e4df9d0bb_f8bcab44974ca57906f4d57bd4289b93, store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:00,362 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c1ac58eccf4444ec9be71f8e4df9d0bb_f8bcab44974ca57906f4d57bd4289b93 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:00,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742349_1525 (size=12104) 2024-11-12T06:50:00,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742350_1526 (size=4469) 2024-11-12T06:50:00,370 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8bcab44974ca57906f4d57bd4289b93#A#compaction#442 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:50:00,370 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/ac3cb6da82a44fb995e034566ec4dbfc is 175, key is test_row_0/A:col10/1731394199468/Put/seqid=0 2024-11-12T06:50:00,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742351_1527 (size=31058) 2024-11-12T06:50:00,384 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/ac3cb6da82a44fb995e034566ec4dbfc as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/ac3cb6da82a44fb995e034566ec4dbfc 2024-11-12T06:50:00,388 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8bcab44974ca57906f4d57bd4289b93/A of f8bcab44974ca57906f4d57bd4289b93 into ac3cb6da82a44fb995e034566ec4dbfc(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:50:00,388 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:00,388 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., storeName=f8bcab44974ca57906f4d57bd4289b93/A, priority=13, startTime=1731394200343; duration=0sec 2024-11-12T06:50:00,388 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:50:00,388 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8bcab44974ca57906f4d57bd4289b93:A 2024-11-12T06:50:00,388 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:50:00,389 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:50:00,389 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): f8bcab44974ca57906f4d57bd4289b93/C is initiating minor compaction (all files) 2024-11-12T06:50:00,389 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8bcab44974ca57906f4d57bd4289b93/C in TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:00,389 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/9168c3718c5244ca873721b493286347, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/f12ff1de4deb4e988b0a633965dec541, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/e93a1f87da0e4ba1aed06eb3a93d2335] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp, totalSize=35.2 K 2024-11-12T06:50:00,389 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9168c3718c5244ca873721b493286347, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731394196634 2024-11-12T06:50:00,390 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting f12ff1de4deb4e988b0a633965dec541, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731394196680 2024-11-12T06:50:00,390 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting e93a1f87da0e4ba1aed06eb3a93d2335, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731394198840 2024-11-12T06:50:00,395 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8bcab44974ca57906f4d57bd4289b93#C#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:50:00,396 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/11c0797fbd6c49b9b115980903832157 is 50, key is test_row_0/C:col10/1731394199468/Put/seqid=0 2024-11-12T06:50:00,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742352_1528 (size=12104) 2024-11-12T06:50:00,402 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/11c0797fbd6c49b9b115980903832157 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/11c0797fbd6c49b9b115980903832157 2024-11-12T06:50:00,406 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8bcab44974ca57906f4d57bd4289b93/C of f8bcab44974ca57906f4d57bd4289b93 into 11c0797fbd6c49b9b115980903832157(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:50:00,406 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:00,406 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., storeName=f8bcab44974ca57906f4d57bd4289b93/C, priority=13, startTime=1731394200343; duration=0sec 2024-11-12T06:50:00,407 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:00,407 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8bcab44974ca57906f4d57bd4289b93:C 2024-11-12T06:50:00,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:00,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8bcab44974ca57906f4d57bd4289b93 3/3 column families, dataSize=131.55 KB heapSize=374.30 KB 2024-11-12T06:50:00,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=A 2024-11-12T06:50:00,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:50:00,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=B 2024-11-12T06:50:00,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:50:00,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=C 2024-11-12T06:50:00,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:50:00,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111290db2ced9cab4989b20639d35d347e8d_f8bcab44974ca57906f4d57bd4289b93 is 50, key is test_row_0/A:col10/1731394199545/Put/seqid=0 2024-11-12T06:50:00,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742353_1529 (size=14594) 2024-11-12T06:50:00,663 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:00,666 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111290db2ced9cab4989b20639d35d347e8d_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111290db2ced9cab4989b20639d35d347e8d_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:00,666 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/63cbad3a0c3a4fdbbfd4bb72db3e34eb, store: [table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:00,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/63cbad3a0c3a4fdbbfd4bb72db3e34eb is 175, key is test_row_0/A:col10/1731394199545/Put/seqid=0 2024-11-12T06:50:00,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394260668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:00,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:00,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394260669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:00,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:00,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742354_1530 (size=39549) 2024-11-12T06:50:00,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394260687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:00,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394260697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:00,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:00,772 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/679ea1a922cd457fb7904fb3b8bf1b29 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/679ea1a922cd457fb7904fb3b8bf1b29 2024-11-12T06:50:00,776 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8bcab44974ca57906f4d57bd4289b93/B of f8bcab44974ca57906f4d57bd4289b93 into 679ea1a922cd457fb7904fb3b8bf1b29(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:50:00,776 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:00,776 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., storeName=f8bcab44974ca57906f4d57bd4289b93/B, priority=13, startTime=1731394200343; duration=0sec 2024-11-12T06:50:00,776 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:00,776 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8bcab44974ca57906f4d57bd4289b93:B 2024-11-12T06:50:00,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394260777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:00,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:00,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394260777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:00,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:00,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T06:50:00,788 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:50:00,788 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:50:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-12T06:50:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T06:50:00,790 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:50:00,790 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:50:00,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:50:00,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:00,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731394260867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:00,868 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_0' in f8bcab44974ca57906f4d57bd4289b93 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:50:00,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T06:50:00,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-12T06:50:00,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:00,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:00,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:00,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:00,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:00,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394260987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:00,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394260987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:00,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:01,082 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/63cbad3a0c3a4fdbbfd4bb72db3e34eb 2024-11-12T06:50:01,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/2ffbe5d268554fadb607b91e99d3a23a is 50, key is test_row_0/B:col10/1731394199545/Put/seqid=0 2024-11-12T06:50:01,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742355_1531 (size=12001) 2024-11-12T06:50:01,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/2ffbe5d268554fadb607b91e99d3a23a 2024-11-12T06:50:01,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-12T06:50:01,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:01,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:01,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:01,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:01,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:01,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:01,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/c56f21fb1c114803bd79f4aa58484210 is 50, key is test_row_0/C:col10/1731394199545/Put/seqid=0 2024-11-12T06:50:01,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742356_1532 (size=12001) 2024-11-12T06:50:01,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T06:50:01,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/c56f21fb1c114803bd79f4aa58484210 2024-11-12T06:50:01,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/63cbad3a0c3a4fdbbfd4bb72db3e34eb as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/63cbad3a0c3a4fdbbfd4bb72db3e34eb 2024-11-12T06:50:01,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/63cbad3a0c3a4fdbbfd4bb72db3e34eb, entries=200, sequenceid=99, filesize=38.6 K 2024-11-12T06:50:01,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/2ffbe5d268554fadb607b91e99d3a23a as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/2ffbe5d268554fadb607b91e99d3a23a 2024-11-12T06:50:01,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/2ffbe5d268554fadb607b91e99d3a23a, entries=150, sequenceid=99, filesize=11.7 K 2024-11-12T06:50:01,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/c56f21fb1c114803bd79f4aa58484210 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/c56f21fb1c114803bd79f4aa58484210 2024-11-12T06:50:01,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/c56f21fb1c114803bd79f4aa58484210, entries=150, sequenceid=99, filesize=11.7 K 2024-11-12T06:50:01,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~131.55 KB/134705, heapSize ~374.26 KB/383240, currentSize=46.96 KB/48090 for f8bcab44974ca57906f4d57bd4289b93 in 480ms, sequenceid=99, compaction requested=false 2024-11-12T06:50:01,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:01,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-12T06:50:01,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:01,247 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing f8bcab44974ca57906f4d57bd4289b93 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T06:50:01,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=A 2024-11-12T06:50:01,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:01,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=B 2024-11-12T06:50:01,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:01,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=C 2024-11-12T06:50:01,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:01,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112da9313a00ad24a92be28c8d912d470b2_f8bcab44974ca57906f4d57bd4289b93 is 50, key is test_row_0/A:col10/1731394200661/Put/seqid=0 2024-11-12T06:50:01,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742357_1533 (size=12154) 2024-11-12T06:50:01,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:01,264 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112da9313a00ad24a92be28c8d912d470b2_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112da9313a00ad24a92be28c8d912d470b2_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:01,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/e6a65b373dbc4caf887d17b14f6be391, store: [table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:01,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/e6a65b373dbc4caf887d17b14f6be391 is 175, key is test_row_0/A:col10/1731394200661/Put/seqid=0 2024-11-12T06:50:01,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742358_1534 (size=30955) 2024-11-12T06:50:01,273 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/e6a65b373dbc4caf887d17b14f6be391 2024-11-12T06:50:01,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/f4079bd11aa14139adc725a19eaf4dab is 50, key is test_row_0/B:col10/1731394200661/Put/seqid=0 2024-11-12T06:50:01,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742359_1535 (size=12001) 2024-11-12T06:50:01,285 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/f4079bd11aa14139adc725a19eaf4dab 2024-11-12T06:50:01,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/cfaf3879cf5d4a9e909c4445d6e0e2cf is 50, key is test_row_0/C:col10/1731394200661/Put/seqid=0 2024-11-12T06:50:01,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742360_1536 (size=12001) 2024-11-12T06:50:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:01,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:01,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394201315, max timestamp=1731394201340Num uniques 859; 2024-11-12T06:50:01,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:50:01,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394201315, max timestamp=1731394201340Num uniques 859; 2024-11-12T06:50:01,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:50:01,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731394201315, max timestamp=1731394201340Num uniques 859; 2024-11-12T06:50:01,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:50:01,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394261383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:01,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394261387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T06:50:01,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394261496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:01,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394261496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:01,696 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/cfaf3879cf5d4a9e909c4445d6e0e2cf 2024-11-12T06:50:01,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/e6a65b373dbc4caf887d17b14f6be391 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/e6a65b373dbc4caf887d17b14f6be391 2024-11-12T06:50:01,702 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/e6a65b373dbc4caf887d17b14f6be391, entries=150, sequenceid=110, filesize=30.2 K 2024-11-12T06:50:01,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/f4079bd11aa14139adc725a19eaf4dab as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/f4079bd11aa14139adc725a19eaf4dab 2024-11-12T06:50:01,705 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/f4079bd11aa14139adc725a19eaf4dab, entries=150, sequenceid=110, filesize=11.7 K 2024-11-12T06:50:01,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/cfaf3879cf5d4a9e909c4445d6e0e2cf as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/cfaf3879cf5d4a9e909c4445d6e0e2cf 2024-11-12T06:50:01,709 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/cfaf3879cf5d4a9e909c4445d6e0e2cf, entries=150, sequenceid=110, filesize=11.7 K 2024-11-12T06:50:01,709 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f8bcab44974ca57906f4d57bd4289b93 in 462ms, sequenceid=110, compaction requested=true 2024-11-12T06:50:01,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:01,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:01,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-12T06:50:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-12T06:50:01,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-12T06:50:01,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 920 msec 2024-11-12T06:50:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:01,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8bcab44974ca57906f4d57bd4289b93 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-12T06:50:01,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731394201334, max timestamp=1731394201712Num uniques 1068; 2024-11-12T06:50:01,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T06:50:01,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731394201334, max timestamp=1731394201712Num uniques 1068; 2024-11-12T06:50:01,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T06:50:01,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731394201334, max timestamp=1731394201712Num uniques 1068; 2024-11-12T06:50:01,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T06:50:01,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 924 msec 2024-11-12T06:50:01,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=A 2024-11-12T06:50:01,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:50:01,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=B 2024-11-12T06:50:01,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:50:01,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=C 2024-11-12T06:50:01,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:50:01,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123e1494e0fe8041ada8b8c3bb4ec21043_f8bcab44974ca57906f4d57bd4289b93 is 50, key is test_row_0/A:col10/1731394201381/Put/seqid=0 2024-11-12T06:50:01,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742361_1537 (size=12304) 2024-11-12T06:50:01,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731394201740, max timestamp=1731394201747Num uniques 859; 2024-11-12T06:50:01,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731394201740, max timestamp=1731394201747Num uniques 859; 2024-11-12T06:50:01,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T06:50:01,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T06:50:01,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731394201740, max timestamp=1731394201747Num uniques 859; 2024-11-12T06:50:01,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T06:50:01,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731394201768, max timestamp=1731394201778Num uniques 1118; 2024-11-12T06:50:01,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:01,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-12T06:50:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394261775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:01,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731394201768, max timestamp=1731394201778Num uniques 1118; 2024-11-12T06:50:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:01,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-12T06:50:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:01,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731394201768, max timestamp=1731394201778Num uniques 1118; 2024-11-12T06:50:01,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-12T06:50:01,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731394201808, max timestamp=1731394201819Num uniques 1077; 2024-11-12T06:50:01,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-12T06:50:01,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731394201808, max timestamp=1731394201819Num uniques 1077; 2024-11-12T06:50:01,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-12T06:50:01,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731394201808, max timestamp=1731394201819Num uniques 1077; 2024-11-12T06:50:01,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-12T06:50:01,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731394201869, max timestamp=1731394201885Num uniques 1036; 2024-11-12T06:50:01,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-12T06:50:01,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731394201869, max timestamp=1731394201885Num uniques 1036; 2024-11-12T06:50:01,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-12T06:50:01,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731394201869, max timestamp=1731394201885Num uniques 1036; 2024-11-12T06:50:01,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-12T06:50:01,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731394201913, max timestamp=1731394201920Num uniques 1095; 2024-11-12T06:50:01,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-12T06:50:01,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731394201913, max timestamp=1731394201920Num uniques 1095; 2024-11-12T06:50:01,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-12T06:50:01,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731394201913, max timestamp=1731394201920Num uniques 1095; 2024-11-12T06:50:01,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-12T06:50:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T06:50:01,928 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:50:01,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:50:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-12T06:50:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T06:50:01,930 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:50:01,930 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:50:01,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:50:01,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731394201940, max timestamp=1731394201956Num uniques 1054; 2024-11-12T06:50:01,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-12T06:50:01,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731394201940, max timestamp=1731394201956Num uniques 1054; 2024-11-12T06:50:01,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731394201940, max timestamp=1731394201956Num uniques 1054; 2024-11-12T06:50:01,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-12T06:50:01,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-12T06:50:01,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731394201975, max timestamp=1731394201995Num uniques 1013; 2024-11-12T06:50:01,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-12T06:50:01,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731394201975, max timestamp=1731394201995Num uniques 1013; 2024-11-12T06:50:01,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-12T06:50:01,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731394201975, max timestamp=1731394201995Num uniques 1013; 2024-11-12T06:50:01,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-12T06:50:02,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731394202019, max timestamp=1731394202028Num uniques 1072; 2024-11-12T06:50:02,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-12T06:50:02,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731394202019, max timestamp=1731394202028Num uniques 1072; 2024-11-12T06:50:02,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-12T06:50:02,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731394202019, max timestamp=1731394202028Num uniques 1072; 2024-11-12T06:50:02,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-12T06:50:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T06:50:02,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731394202046, max timestamp=1731394202064Num uniques 1131; 2024-11-12T06:50:02,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731394202046, max timestamp=1731394202064Num uniques 1131; 2024-11-12T06:50:02,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731394202046, max timestamp=1731394202064Num uniques 1131; 2024-11-12T06:50:02,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=12866, suffix heap size=145592,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=281 2024-11-12T06:50:02,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=12866, suffix heap size=145592,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=281 2024-11-12T06:50:02,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=12866, suffix heap size=145592,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=281 2024-11-12T06:50:02,072 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T06:50:02,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T06:50:02,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:02,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:02,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:02,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:02,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:02,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394262096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:02,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:02,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731394202090, max timestamp=1731394202100Num uniques 1140; 2024-11-12T06:50:02,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=13280, suffix heap size=146528,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=290 2024-11-12T06:50:02,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731394202090, max timestamp=1731394202100Num uniques 1140; 2024-11-12T06:50:02,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731394202090, max timestamp=1731394202100Num uniques 1140; 2024-11-12T06:50:02,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=13280, suffix heap size=146528,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=290 2024-11-12T06:50:02,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=13280, suffix heap size=146528,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=290 2024-11-12T06:50:02,133 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:02,135 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123e1494e0fe8041ada8b8c3bb4ec21043_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123e1494e0fe8041ada8b8c3bb4ec21043_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:02,136 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/17a6d5f2326f40d194eb2c98ac76ef83, store: [table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:02,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/17a6d5f2326f40d194eb2c98ac76ef83 is 175, key is test_row_0/A:col10/1731394201381/Put/seqid=0 2024-11-12T06:50:02,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742362_1538 (size=31105) 2024-11-12T06:50:02,147 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=148, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/17a6d5f2326f40d194eb2c98ac76ef83 2024-11-12T06:50:02,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731394202128, max timestamp=1731394202147Num uniques 1149; 2024-11-12T06:50:02,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=13694, suffix heap size=147464,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=299 2024-11-12T06:50:02,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731394202128, max timestamp=1731394202147Num uniques 1149; 2024-11-12T06:50:02,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=13694, suffix heap size=147464,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=299 2024-11-12T06:50:02,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731394202128, max timestamp=1731394202147Num uniques 1149; 2024-11-12T06:50:02,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=13694, suffix heap size=147464,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=299 2024-11-12T06:50:02,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/8bf8b4d30d3c44ce9903d891a373f38e is 50, key is test_row_0/B:col10/1731394201381/Put/seqid=0 2024-11-12T06:50:02,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742363_1539 (size=12151) 2024-11-12T06:50:02,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/8bf8b4d30d3c44ce9903d891a373f38e 2024-11-12T06:50:02,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/b2ddde96b1c24a4aac12478bf2506568 is 50, key is test_row_0/C:col10/1731394201381/Put/seqid=0 2024-11-12T06:50:02,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742364_1540 (size=12151) 2024-11-12T06:50:02,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:02,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394262190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:02,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:02,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394262190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:02,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:02,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:02,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394262191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:02,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:02,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:02,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:02,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394262207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:02,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T06:50:02,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:02,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:02,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:02,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:02,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:02,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T06:50:02,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:02,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394262307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:02,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:02,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394262307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:02,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:02,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394262307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:02,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T06:50:02,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:02,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:02,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:02,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:02,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:02,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:02,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394262417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:02,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:02,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:02,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394262517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:02,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394262517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:02,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:02,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394262527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:02,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:02,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T06:50:02,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:02,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:02,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:02,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:02,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:02,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:02,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T06:50:02,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/b2ddde96b1c24a4aac12478bf2506568 2024-11-12T06:50:02,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/17a6d5f2326f40d194eb2c98ac76ef83 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/17a6d5f2326f40d194eb2c98ac76ef83 2024-11-12T06:50:02,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/17a6d5f2326f40d194eb2c98ac76ef83, entries=150, sequenceid=148, filesize=30.4 K 2024-11-12T06:50:02,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/8bf8b4d30d3c44ce9903d891a373f38e as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/8bf8b4d30d3c44ce9903d891a373f38e 2024-11-12T06:50:02,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/8bf8b4d30d3c44ce9903d891a373f38e, entries=150, sequenceid=148, filesize=11.9 K 2024-11-12T06:50:02,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/b2ddde96b1c24a4aac12478bf2506568 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/b2ddde96b1c24a4aac12478bf2506568 2024-11-12T06:50:02,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/b2ddde96b1c24a4aac12478bf2506568, entries=150, sequenceid=148, filesize=11.9 K 2024-11-12T06:50:02,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~80.58 KB/82512, currentSize=154.31 KB/158010 for f8bcab44974ca57906f4d57bd4289b93 in 896ms, sequenceid=148, compaction requested=true 2024-11-12T06:50:02,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:02,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8bcab44974ca57906f4d57bd4289b93:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:50:02,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:02,608 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:50:02,608 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:50:02,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8bcab44974ca57906f4d57bd4289b93:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:50:02,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:02,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8bcab44974ca57906f4d57bd4289b93:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:50:02,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:50:02,612 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:50:02,612 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132667 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:50:02,612 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): f8bcab44974ca57906f4d57bd4289b93/B is initiating minor compaction (all files) 2024-11-12T06:50:02,612 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): f8bcab44974ca57906f4d57bd4289b93/A is initiating minor compaction (all files) 2024-11-12T06:50:02,612 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8bcab44974ca57906f4d57bd4289b93/A in TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:02,612 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8bcab44974ca57906f4d57bd4289b93/B in TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:02,612 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/679ea1a922cd457fb7904fb3b8bf1b29, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/2ffbe5d268554fadb607b91e99d3a23a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/f4079bd11aa14139adc725a19eaf4dab, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/8bf8b4d30d3c44ce9903d891a373f38e] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp, totalSize=47.1 K 2024-11-12T06:50:02,612 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/ac3cb6da82a44fb995e034566ec4dbfc, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/63cbad3a0c3a4fdbbfd4bb72db3e34eb, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/e6a65b373dbc4caf887d17b14f6be391, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/17a6d5f2326f40d194eb2c98ac76ef83] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp, totalSize=129.6 K 2024-11-12T06:50:02,612 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:02,612 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. files: [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/ac3cb6da82a44fb995e034566ec4dbfc, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/63cbad3a0c3a4fdbbfd4bb72db3e34eb, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/e6a65b373dbc4caf887d17b14f6be391, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/17a6d5f2326f40d194eb2c98ac76ef83] 2024-11-12T06:50:02,612 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 679ea1a922cd457fb7904fb3b8bf1b29, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731394198840 2024-11-12T06:50:02,612 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac3cb6da82a44fb995e034566ec4dbfc, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731394198840 2024-11-12T06:50:02,613 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ffbe5d268554fadb607b91e99d3a23a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731394199545 2024-11-12T06:50:02,613 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63cbad3a0c3a4fdbbfd4bb72db3e34eb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731394199542 2024-11-12T06:50:02,613 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6a65b373dbc4caf887d17b14f6be391, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731394200660 2024-11-12T06:50:02,613 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting f4079bd11aa14139adc725a19eaf4dab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731394200660 2024-11-12T06:50:02,613 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17a6d5f2326f40d194eb2c98ac76ef83, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731394201381 2024-11-12T06:50:02,614 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bf8b4d30d3c44ce9903d891a373f38e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731394201381 2024-11-12T06:50:02,623 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8bcab44974ca57906f4d57bd4289b93#B#compaction#453 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:50:02,623 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/57bc0625deda4ff4ad72e6fdbc78cb6b is 50, key is test_row_0/B:col10/1731394201381/Put/seqid=0 2024-11-12T06:50:02,635 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:02,643 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112ba28eca963be4f03bd874c187e263305_f8bcab44974ca57906f4d57bd4289b93 store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:02,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742365_1541 (size=12391) 2024-11-12T06:50:02,645 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112ba28eca963be4f03bd874c187e263305_f8bcab44974ca57906f4d57bd4289b93, store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:02,645 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ba28eca963be4f03bd874c187e263305_f8bcab44974ca57906f4d57bd4289b93 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:02,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742366_1542 (size=4469) 2024-11-12T06:50:02,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T06:50:02,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:02,693 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing f8bcab44974ca57906f4d57bd4289b93 3/3 column families, dataSize=154.31 KB heapSize=431.63 KB 2024-11-12T06:50:02,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=A 2024-11-12T06:50:02,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:50:02,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=B 2024-11-12T06:50:02,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:50:02,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=C 2024-11-12T06:50:02,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:50:02,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112fae37d2f50f942c78b4dbbaa08cf0179_f8bcab44974ca57906f4d57bd4289b93 is 50, key is test_row_0/A:col10/1731394202159/Put/seqid=0 2024-11-12T06:50:02,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742367_1543 (size=12454) 2024-11-12T06:50:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:02,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:02,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394262750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:02,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:02,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394262827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:02,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:02,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394262827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:02,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394262836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:02,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:02,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:02,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394262867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:02,872 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,872 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:02,872 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:03,050 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/57bc0625deda4ff4ad72e6fdbc78cb6b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/57bc0625deda4ff4ad72e6fdbc78cb6b 2024-11-12T06:50:03,054 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8bcab44974ca57906f4d57bd4289b93#A#compaction#454 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:50:03,054 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/efc1a96a8eac4a298139c2a31290d9df is 175, key is test_row_0/A:col10/1731394201381/Put/seqid=0 2024-11-12T06:50:03,055 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8bcab44974ca57906f4d57bd4289b93/B of f8bcab44974ca57906f4d57bd4289b93 into 57bc0625deda4ff4ad72e6fdbc78cb6b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:50:03,055 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:03,055 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., storeName=f8bcab44974ca57906f4d57bd4289b93/B, priority=12, startTime=1731394202608; duration=0sec 2024-11-12T06:50:03,055 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:50:03,055 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8bcab44974ca57906f4d57bd4289b93:B 2024-11-12T06:50:03,055 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:50:03,056 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:50:03,056 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): f8bcab44974ca57906f4d57bd4289b93/C is initiating minor compaction (all files) 2024-11-12T06:50:03,056 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8bcab44974ca57906f4d57bd4289b93/C in TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:03,056 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/11c0797fbd6c49b9b115980903832157, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/c56f21fb1c114803bd79f4aa58484210, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/cfaf3879cf5d4a9e909c4445d6e0e2cf, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/b2ddde96b1c24a4aac12478bf2506568] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp, totalSize=47.1 K 2024-11-12T06:50:03,057 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 11c0797fbd6c49b9b115980903832157, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731394198840 2024-11-12T06:50:03,057 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting c56f21fb1c114803bd79f4aa58484210, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731394199545 2024-11-12T06:50:03,057 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting cfaf3879cf5d4a9e909c4445d6e0e2cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731394200660 2024-11-12T06:50:03,057 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting b2ddde96b1c24a4aac12478bf2506568, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731394201381 2024-11-12T06:50:03,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742368_1544 (size=31345) 2024-11-12T06:50:03,064 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/efc1a96a8eac4a298139c2a31290d9df as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/efc1a96a8eac4a298139c2a31290d9df 2024-11-12T06:50:03,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T06:50:03,069 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8bcab44974ca57906f4d57bd4289b93/A of f8bcab44974ca57906f4d57bd4289b93 into efc1a96a8eac4a298139c2a31290d9df(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:50:03,069 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:03,069 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., storeName=f8bcab44974ca57906f4d57bd4289b93/A, priority=12, startTime=1731394202608; duration=0sec 2024-11-12T06:50:03,069 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:03,069 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8bcab44974ca57906f4d57bd4289b93:A 2024-11-12T06:50:03,069 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8bcab44974ca57906f4d57bd4289b93#C#compaction#456 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:50:03,070 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/c8a690584e8848ce94f696e7cbf92d70 is 50, key is test_row_0/C:col10/1731394201381/Put/seqid=0 2024-11-12T06:50:03,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742369_1545 (size=12391) 2024-11-12T06:50:03,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394263077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:03,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:03,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:03,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:03,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:03,107 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112fae37d2f50f942c78b4dbbaa08cf0179_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112fae37d2f50f942c78b4dbbaa08cf0179_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:03,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/29ba59bb13cb4b209cad124846308daf, store: [table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:03,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/29ba59bb13cb4b209cad124846308daf is 175, key is test_row_0/A:col10/1731394202159/Put/seqid=0 2024-11-12T06:50:03,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742370_1546 (size=31255) 2024-11-12T06:50:03,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394263337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:03,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394263337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:03,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:03,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394263346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:03,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:03,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:03,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:03,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:03,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394263387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:03,390 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:03,390 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:03,390 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:03,477 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/c8a690584e8848ce94f696e7cbf92d70 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/c8a690584e8848ce94f696e7cbf92d70 2024-11-12T06:50:03,481 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8bcab44974ca57906f4d57bd4289b93/C of f8bcab44974ca57906f4d57bd4289b93 into c8a690584e8848ce94f696e7cbf92d70(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:50:03,481 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:03,481 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., storeName=f8bcab44974ca57906f4d57bd4289b93/C, priority=12, startTime=1731394202608; duration=0sec 2024-11-12T06:50:03,481 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:03,481 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8bcab44974ca57906f4d57bd4289b93:C 2024-11-12T06:50:03,511 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=357, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/29ba59bb13cb4b209cad124846308daf 2024-11-12T06:50:03,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/0b6eaccb80974eb0a99fbeb4b9ff64d9 is 50, key is test_row_0/B:col10/1731394202159/Put/seqid=0 2024-11-12T06:50:03,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742371_1547 (size=12301) 2024-11-12T06:50:03,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:03,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394263897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:03,921 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/0b6eaccb80974eb0a99fbeb4b9ff64d9 2024-11-12T06:50:03,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/41a9c83c3e0248d580d9d07868660225 is 50, key is test_row_0/C:col10/1731394202159/Put/seqid=0 2024-11-12T06:50:03,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742372_1548 (size=12301) 2024-11-12T06:50:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T06:50:04,330 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/41a9c83c3e0248d580d9d07868660225 2024-11-12T06:50:04,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/29ba59bb13cb4b209cad124846308daf as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/29ba59bb13cb4b209cad124846308daf 2024-11-12T06:50:04,337 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/29ba59bb13cb4b209cad124846308daf, entries=150, sequenceid=357, filesize=30.5 K 2024-11-12T06:50:04,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/0b6eaccb80974eb0a99fbeb4b9ff64d9 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/0b6eaccb80974eb0a99fbeb4b9ff64d9 2024-11-12T06:50:04,340 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/0b6eaccb80974eb0a99fbeb4b9ff64d9, entries=150, sequenceid=357, filesize=12.0 K 2024-11-12T06:50:04,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/41a9c83c3e0248d580d9d07868660225 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/41a9c83c3e0248d580d9d07868660225 2024-11-12T06:50:04,344 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/41a9c83c3e0248d580d9d07868660225, entries=150, sequenceid=357, filesize=12.0 K 2024-11-12T06:50:04,344 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~431.58 KB/441936, currentSize=26.84 KB/27480 for f8bcab44974ca57906f4d57bd4289b93 in 1651ms, sequenceid=357, compaction requested=false 2024-11-12T06:50:04,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:04,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:04,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-12T06:50:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-12T06:50:04,348 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-12T06:50:04,348 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4160 sec 2024-11-12T06:50:04,350 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.4210 sec 2024-11-12T06:50:04,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:04,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8bcab44974ca57906f4d57bd4289b93 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T06:50:04,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=A 2024-11-12T06:50:04,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:04,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=B 2024-11-12T06:50:04,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:04,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=C 2024-11-12T06:50:04,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:04,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111286b77b95ef694589b9a04f59e26b0435_f8bcab44974ca57906f4d57bd4289b93 is 50, key is test_row_0/A:col10/1731394204350/Put/seqid=0 2024-11-12T06:50:04,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742373_1549 (size=14994) 2024-11-12T06:50:04,381 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:04,384 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111286b77b95ef694589b9a04f59e26b0435_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111286b77b95ef694589b9a04f59e26b0435_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:04,384 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/f9a7728de5b2438b9b6dc2dc9288d5b5, store: [table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:04,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/f9a7728de5b2438b9b6dc2dc9288d5b5 is 175, key is test_row_0/A:col10/1731394204350/Put/seqid=0 2024-11-12T06:50:04,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742374_1550 (size=39949) 2024-11-12T06:50:04,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:04,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394264426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:04,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394204382, max timestamp=1731394204430Num uniques 859; 2024-11-12T06:50:04,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:50:04,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394204382, max timestamp=1731394204430Num uniques 859; 2024-11-12T06:50:04,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394204382, max timestamp=1731394204430Num uniques 859; 2024-11-12T06:50:04,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:50:04,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:50:04,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:04,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:04,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:04,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394264476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:04,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:04,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:04,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:04,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394264478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:04,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:04,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394264537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:04,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:04,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:04,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:04,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394264587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:04,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394264587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:04,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:04,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394264747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:04,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:04,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:04,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:04,788 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/f9a7728de5b2438b9b6dc2dc9288d5b5 2024-11-12T06:50:04,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/253c35951d3b4bd8a9f0efa3cab40b15 is 50, key is test_row_0/B:col10/1731394204350/Put/seqid=0 2024-11-12T06:50:04,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742375_1551 (size=12301) 2024-11-12T06:50:04,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/253c35951d3b4bd8a9f0efa3cab40b15 2024-11-12T06:50:04,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:04,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394264797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:04,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:04,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394264797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:04,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/2e8eae894e3c43a882c1f738cb4f95d8 is 50, key is test_row_0/C:col10/1731394204350/Put/seqid=0 2024-11-12T06:50:04,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742376_1552 (size=12301) 2024-11-12T06:50:04,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:04,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731394264897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:04,899 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_0' in f8bcab44974ca57906f4d57bd4289b93 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:50:04,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:04,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:04,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:04,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:04,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394264907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:04,910 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_0' in f8bcab44974ca57906f4d57bd4289b93 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:50:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:05,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:05,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394265057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:05,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:05,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:05,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:05,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394265107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:05,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394265107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:05,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/2e8eae894e3c43a882c1f738cb4f95d8 2024-11-12T06:50:05,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/f9a7728de5b2438b9b6dc2dc9288d5b5 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/f9a7728de5b2438b9b6dc2dc9288d5b5 2024-11-12T06:50:05,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/f9a7728de5b2438b9b6dc2dc9288d5b5, entries=200, sequenceid=370, filesize=39.0 K 2024-11-12T06:50:05,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/253c35951d3b4bd8a9f0efa3cab40b15 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/253c35951d3b4bd8a9f0efa3cab40b15 2024-11-12T06:50:05,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/253c35951d3b4bd8a9f0efa3cab40b15, entries=150, sequenceid=370, filesize=12.0 K 2024-11-12T06:50:05,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/2e8eae894e3c43a882c1f738cb4f95d8 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/2e8eae894e3c43a882c1f738cb4f95d8 2024-11-12T06:50:05,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/2e8eae894e3c43a882c1f738cb4f95d8, entries=150, sequenceid=370, filesize=12.0 K 2024-11-12T06:50:05,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f8bcab44974ca57906f4d57bd4289b93 in 866ms, sequenceid=370, compaction requested=true 2024-11-12T06:50:05,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:05,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8bcab44974ca57906f4d57bd4289b93:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:50:05,221 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:50:05,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:05,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8bcab44974ca57906f4d57bd4289b93:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:50:05,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:05,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8bcab44974ca57906f4d57bd4289b93:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:50:05,221 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:50:05,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:50:05,221 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:50:05,222 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:50:05,222 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): f8bcab44974ca57906f4d57bd4289b93/A is initiating minor compaction (all files) 2024-11-12T06:50:05,222 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): f8bcab44974ca57906f4d57bd4289b93/B is initiating minor compaction (all files) 2024-11-12T06:50:05,222 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8bcab44974ca57906f4d57bd4289b93/B in TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:05,222 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8bcab44974ca57906f4d57bd4289b93/A in TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:05,222 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/57bc0625deda4ff4ad72e6fdbc78cb6b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/0b6eaccb80974eb0a99fbeb4b9ff64d9, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/253c35951d3b4bd8a9f0efa3cab40b15] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp, totalSize=36.1 K 2024-11-12T06:50:05,222 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/efc1a96a8eac4a298139c2a31290d9df, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/29ba59bb13cb4b209cad124846308daf, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/f9a7728de5b2438b9b6dc2dc9288d5b5] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp, totalSize=100.1 K 2024-11-12T06:50:05,222 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:05,222 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. files: [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/efc1a96a8eac4a298139c2a31290d9df, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/29ba59bb13cb4b209cad124846308daf, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/f9a7728de5b2438b9b6dc2dc9288d5b5] 2024-11-12T06:50:05,222 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 57bc0625deda4ff4ad72e6fdbc78cb6b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731394201381 2024-11-12T06:50:05,222 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting efc1a96a8eac4a298139c2a31290d9df, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731394201381 2024-11-12T06:50:05,222 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b6eaccb80974eb0a99fbeb4b9ff64d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731394202159 2024-11-12T06:50:05,223 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29ba59bb13cb4b209cad124846308daf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731394202159 2024-11-12T06:50:05,223 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 253c35951d3b4bd8a9f0efa3cab40b15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731394202738 2024-11-12T06:50:05,223 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9a7728de5b2438b9b6dc2dc9288d5b5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731394202738 2024-11-12T06:50:05,228 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:05,228 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8bcab44974ca57906f4d57bd4289b93#B#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:50:05,229 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/757aff8019894c89aee6b9cbbe6066f4 is 50, key is test_row_0/B:col10/1731394204350/Put/seqid=0 2024-11-12T06:50:05,231 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111269c06f00841a4d21a89e823c04aaf21a_f8bcab44974ca57906f4d57bd4289b93 store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:05,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742377_1553 (size=12643) 2024-11-12T06:50:05,233 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111269c06f00841a4d21a89e823c04aaf21a_f8bcab44974ca57906f4d57bd4289b93, store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:05,233 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111269c06f00841a4d21a89e823c04aaf21a_f8bcab44974ca57906f4d57bd4289b93 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:05,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742378_1554 (size=4469) 2024-11-12T06:50:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:05,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8bcab44974ca57906f4d57bd4289b93 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-12T06:50:05,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=A 2024-11-12T06:50:05,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:50:05,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=B 2024-11-12T06:50:05,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:50:05,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=C 2024-11-12T06:50:05,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T06:50:05,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ad5ceb27fdc54ef3a3781c10763f0f4d_f8bcab44974ca57906f4d57bd4289b93 is 50, key is test_row_0/A:col10/1731394205569/Put/seqid=0 2024-11-12T06:50:05,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742379_1555 (size=12454) 2024-11-12T06:50:05,590 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:05,593 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ad5ceb27fdc54ef3a3781c10763f0f4d_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ad5ceb27fdc54ef3a3781c10763f0f4d_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:05,593 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/a1a5dc9c667b41e392a33228a1b0258e, store: [table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:05,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/a1a5dc9c667b41e392a33228a1b0258e is 175, key is test_row_0/A:col10/1731394205569/Put/seqid=0 2024-11-12T06:50:05,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742380_1556 (size=31255) 2024-11-12T06:50:05,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394265609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:05,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:05,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:05,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:05,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394265617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:05,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394265627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:05,636 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/757aff8019894c89aee6b9cbbe6066f4 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/757aff8019894c89aee6b9cbbe6066f4 2024-11-12T06:50:05,637 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8bcab44974ca57906f4d57bd4289b93#A#compaction#463 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:50:05,637 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/2f3845f84dc4444796130684a3900109 is 175, key is test_row_0/A:col10/1731394204350/Put/seqid=0 2024-11-12T06:50:05,642 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8bcab44974ca57906f4d57bd4289b93/B of f8bcab44974ca57906f4d57bd4289b93 into 757aff8019894c89aee6b9cbbe6066f4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:50:05,642 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:05,642 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., storeName=f8bcab44974ca57906f4d57bd4289b93/B, priority=13, startTime=1731394205221; duration=0sec 2024-11-12T06:50:05,642 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:50:05,642 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8bcab44974ca57906f4d57bd4289b93:B 2024-11-12T06:50:05,642 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:50:05,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742381_1557 (size=31597) 2024-11-12T06:50:05,643 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:50:05,643 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): f8bcab44974ca57906f4d57bd4289b93/C is initiating minor compaction (all files) 2024-11-12T06:50:05,643 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8bcab44974ca57906f4d57bd4289b93/C in TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:05,643 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/c8a690584e8848ce94f696e7cbf92d70, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/41a9c83c3e0248d580d9d07868660225, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/2e8eae894e3c43a882c1f738cb4f95d8] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp, totalSize=36.1 K 2024-11-12T06:50:05,644 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting c8a690584e8848ce94f696e7cbf92d70, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731394201381 2024-11-12T06:50:05,644 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 41a9c83c3e0248d580d9d07868660225, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731394202159 2024-11-12T06:50:05,644 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e8eae894e3c43a882c1f738cb4f95d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731394202738 2024-11-12T06:50:05,648 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/2f3845f84dc4444796130684a3900109 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/2f3845f84dc4444796130684a3900109 2024-11-12T06:50:05,651 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8bcab44974ca57906f4d57bd4289b93#C#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:50:05,651 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8bcab44974ca57906f4d57bd4289b93/A of f8bcab44974ca57906f4d57bd4289b93 into 2f3845f84dc4444796130684a3900109(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:50:05,651 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:05,651 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., storeName=f8bcab44974ca57906f4d57bd4289b93/A, priority=13, startTime=1731394205221; duration=0sec 2024-11-12T06:50:05,651 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/92afd40a2a274471a8f084648f4cc716 is 50, key is test_row_0/C:col10/1731394204350/Put/seqid=0 2024-11-12T06:50:05,651 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:05,651 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8bcab44974ca57906f4d57bd4289b93:A 2024-11-12T06:50:05,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742382_1558 (size=12643) 2024-11-12T06:50:05,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394265727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:05,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394265937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:05,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:05,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:05,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:06,010 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=408, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/a1a5dc9c667b41e392a33228a1b0258e 2024-11-12T06:50:06,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/1470e421fb304d6f81d429814ecbda56 is 50, key is test_row_0/B:col10/1731394205569/Put/seqid=0 2024-11-12T06:50:06,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742383_1559 (size=12301) 2024-11-12T06:50:06,067 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/92afd40a2a274471a8f084648f4cc716 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/92afd40a2a274471a8f084648f4cc716 2024-11-12T06:50:06,071 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8bcab44974ca57906f4d57bd4289b93/C of f8bcab44974ca57906f4d57bd4289b93 into 92afd40a2a274471a8f084648f4cc716(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:50:06,071 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:06,071 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., storeName=f8bcab44974ca57906f4d57bd4289b93/C, priority=13, startTime=1731394205221; duration=0sec 2024-11-12T06:50:06,071 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:06,071 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8bcab44974ca57906f4d57bd4289b93:C 2024-11-12T06:50:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T06:50:06,107 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:50:06,108 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:50:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-12T06:50:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T06:50:06,109 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:50:06,109 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:50:06,110 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:50:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T06:50:06,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394266247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:06,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:06,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:06,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:06,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T06:50:06,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:06,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:06,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:06,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:06,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:06,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T06:50:06,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:06,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:06,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:06,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:06,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:06,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/1470e421fb304d6f81d429814ecbda56 2024-11-12T06:50:06,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/8a2a3ca4ab3b44d788faaf2045a21fdf is 50, key is test_row_0/C:col10/1731394205569/Put/seqid=0 2024-11-12T06:50:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T06:50:06,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742384_1560 (size=12301) 2024-11-12T06:50:06,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/8a2a3ca4ab3b44d788faaf2045a21fdf 2024-11-12T06:50:06,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/a1a5dc9c667b41e392a33228a1b0258e as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/a1a5dc9c667b41e392a33228a1b0258e 2024-11-12T06:50:06,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/a1a5dc9c667b41e392a33228a1b0258e, entries=150, sequenceid=408, filesize=30.5 K 2024-11-12T06:50:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/1470e421fb304d6f81d429814ecbda56 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/1470e421fb304d6f81d429814ecbda56 2024-11-12T06:50:06,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/1470e421fb304d6f81d429814ecbda56, entries=150, sequenceid=408, filesize=12.0 K 2024-11-12T06:50:06,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/8a2a3ca4ab3b44d788faaf2045a21fdf as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/8a2a3ca4ab3b44d788faaf2045a21fdf 2024-11-12T06:50:06,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/8a2a3ca4ab3b44d788faaf2045a21fdf, entries=150, sequenceid=408, filesize=12.0 K 2024-11-12T06:50:06,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for f8bcab44974ca57906f4d57bd4289b93 in 877ms, sequenceid=408, compaction requested=false 2024-11-12T06:50:06,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:06,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T06:50:06,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:06,566 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing f8bcab44974ca57906f4d57bd4289b93 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-12T06:50:06,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=A 2024-11-12T06:50:06,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:06,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=B 2024-11-12T06:50:06,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:06,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=C 2024-11-12T06:50:06,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:06,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112abc10da3d8e54d1485117e401082a4d7_f8bcab44974ca57906f4d57bd4289b93 is 50, key is test_row_0/A:col10/1731394205587/Put/seqid=0 2024-11-12T06:50:06,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742385_1561 (size=12454) 2024-11-12T06:50:06,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:06,589 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112abc10da3d8e54d1485117e401082a4d7_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112abc10da3d8e54d1485117e401082a4d7_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:06,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/899b0a43857b4d6b8dbf9a746c89e190, store: [table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:06,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/899b0a43857b4d6b8dbf9a746c89e190 is 175, key is test_row_0/A:col10/1731394205587/Put/seqid=0 2024-11-12T06:50:06,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742386_1562 (size=31255) 2024-11-12T06:50:06,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:06,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:06,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394206662, max timestamp=1731394206680Num uniques 859; 2024-11-12T06:50:06,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394206662, max timestamp=1731394206680Num uniques 859; 2024-11-12T06:50:06,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:50:06,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:50:06,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:06,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394266712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:06,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:06,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394266713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:06,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:06,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:06,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:06,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:06,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:06,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:06,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T06:50:06,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394266756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:06,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:06,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:06,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:06,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:06,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394266827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:06,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:06,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394266827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:06,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:06,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:06,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:06,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394266927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:06,930 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_0' in f8bcab44974ca57906f4d57bd4289b93 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:50:06,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:06,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:06,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:06,996 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=420, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/899b0a43857b4d6b8dbf9a746c89e190 2024-11-12T06:50:07,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/03533005ff1041c9bb75a507509686e8 is 50, key is test_row_0/B:col10/1731394205587/Put/seqid=0 2024-11-12T06:50:07,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742387_1563 (size=12301) 2024-11-12T06:50:07,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394267037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:07,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:07,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394267037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:07,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:07,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:07,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:07,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:07,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T06:50:07,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394267347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:07,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:07,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:07,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394267348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:07,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:07,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:07,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:07,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:07,405 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/03533005ff1041c9bb75a507509686e8 2024-11-12T06:50:07,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/63af74117c5149b5ab266ef288b21800 is 50, key is test_row_0/C:col10/1731394205587/Put/seqid=0 2024-11-12T06:50:07,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742388_1564 (size=12301) 2024-11-12T06:50:07,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:07,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394267767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:07,768 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_1' in f8bcab44974ca57906f4d57bd4289b93 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:50:07,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:07,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:07,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:07,815 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/63af74117c5149b5ab266ef288b21800 2024-11-12T06:50:07,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/899b0a43857b4d6b8dbf9a746c89e190 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/899b0a43857b4d6b8dbf9a746c89e190 2024-11-12T06:50:07,821 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/899b0a43857b4d6b8dbf9a746c89e190, entries=150, sequenceid=420, filesize=30.5 K 2024-11-12T06:50:07,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/03533005ff1041c9bb75a507509686e8 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/03533005ff1041c9bb75a507509686e8 2024-11-12T06:50:07,824 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/03533005ff1041c9bb75a507509686e8, entries=150, sequenceid=420, filesize=12.0 K 2024-11-12T06:50:07,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/63af74117c5149b5ab266ef288b21800 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/63af74117c5149b5ab266ef288b21800 2024-11-12T06:50:07,827 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/63af74117c5149b5ab266ef288b21800, entries=150, sequenceid=420, filesize=12.0 K 2024-11-12T06:50:07,828 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=136.42 KB/139690 for f8bcab44974ca57906f4d57bd4289b93 in 1263ms, sequenceid=420, compaction requested=true 2024-11-12T06:50:07,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:07,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:07,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-12T06:50:07,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-12T06:50:07,831 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-12T06:50:07,831 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7200 sec 2024-11-12T06:50:07,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.7230 sec 2024-11-12T06:50:07,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:07,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8bcab44974ca57906f4d57bd4289b93 3/3 column families, dataSize=149.83 KB heapSize=433.31 KB 2024-11-12T06:50:07,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=A 2024-11-12T06:50:07,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:50:07,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=B 2024-11-12T06:50:07,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:50:07,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=C 2024-11-12T06:50:07,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:50:07,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111276bc8c412c0d435eb513008536e6c095_f8bcab44974ca57906f4d57bd4289b93 is 50, key is test_row_0/A:col10/1731394206710/Put/seqid=0 2024-11-12T06:50:07,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742389_1565 (size=17534) 2024-11-12T06:50:07,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:07,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394267881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:07,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:07,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:07,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:07,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394267881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:07,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:07,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:07,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:07,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:07,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:07,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394267997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:07,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:08,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:08,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:08,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394267997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:08,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:08,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:08,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:08,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:08,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:08,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394268207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:08,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:08,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:08,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:08,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394268207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:08,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:08,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:08,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:08,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:08,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T06:50:08,257 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:50:08,258 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:50:08,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-12T06:50:08,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T06:50:08,259 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:50:08,260 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:50:08,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:50:08,273 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:08,276 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111276bc8c412c0d435eb513008536e6c095_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111276bc8c412c0d435eb513008536e6c095_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:08,277 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/56416897c9a64a5584e9a9d47383ae7d, store: [table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:08,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/56416897c9a64a5584e9a9d47383ae7d is 175, key is test_row_0/A:col10/1731394206710/Put/seqid=0 2024-11-12T06:50:08,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742390_1566 (size=48639) 2024-11-12T06:50:08,288 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=454, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/56416897c9a64a5584e9a9d47383ae7d 2024-11-12T06:50:08,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/3433444266e4457684a11e7f726601f4 is 50, key is test_row_0/B:col10/1731394206710/Put/seqid=0 2024-11-12T06:50:08,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742391_1567 (size=12301) 2024-11-12T06:50:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T06:50:08,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T06:50:08,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:08,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:08,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:08,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:08,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:08,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:08,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394268517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:08,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:08,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394268517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:08,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T06:50:08,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:08,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:08,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:08,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:08,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:08,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:08,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T06:50:08,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/3433444266e4457684a11e7f726601f4 2024-11-12T06:50:08,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/a4f008844d4f4f3ea9e0368f33265d58 is 50, key is test_row_0/C:col10/1731394206710/Put/seqid=0 2024-11-12T06:50:08,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742392_1568 (size=12301) 2024-11-12T06:50:08,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T06:50:08,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:08,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:08,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:08,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:08,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:08,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:08,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T06:50:08,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:08,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:08,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:08,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:08,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T06:50:09,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T06:50:09,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:09,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:09,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:09,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:09,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:09,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394269027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:09,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:09,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:09,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:09,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394269027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:09,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:09,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:09,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:09,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=69.33 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/a4f008844d4f4f3ea9e0368f33265d58 2024-11-12T06:50:09,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/56416897c9a64a5584e9a9d47383ae7d as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/56416897c9a64a5584e9a9d47383ae7d 2024-11-12T06:50:09,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/56416897c9a64a5584e9a9d47383ae7d, entries=250, sequenceid=454, filesize=47.5 K 2024-11-12T06:50:09,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/3433444266e4457684a11e7f726601f4 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/3433444266e4457684a11e7f726601f4 2024-11-12T06:50:09,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/3433444266e4457684a11e7f726601f4, entries=150, sequenceid=454, filesize=12.0 K 2024-11-12T06:50:09,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/a4f008844d4f4f3ea9e0368f33265d58 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/a4f008844d4f4f3ea9e0368f33265d58 2024-11-12T06:50:09,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/a4f008844d4f4f3ea9e0368f33265d58, entries=150, sequenceid=454, filesize=12.0 K 2024-11-12T06:50:09,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~149.83 KB/153430, heapSize ~433.27 KB/443664, currentSize=26.84 KB/27480 for f8bcab44974ca57906f4d57bd4289b93 in 1265ms, sequenceid=454, compaction requested=true 2024-11-12T06:50:09,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:09,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8bcab44974ca57906f4d57bd4289b93:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:50:09,128 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:50:09,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:09,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8bcab44974ca57906f4d57bd4289b93:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:50:09,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:09,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8bcab44974ca57906f4d57bd4289b93:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:50:09,128 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:50:09,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:50:09,129 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49546 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:50:09,129 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): f8bcab44974ca57906f4d57bd4289b93/B is initiating minor compaction (all files) 2024-11-12T06:50:09,129 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8bcab44974ca57906f4d57bd4289b93/B in TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:09,130 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/757aff8019894c89aee6b9cbbe6066f4, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/1470e421fb304d6f81d429814ecbda56, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/03533005ff1041c9bb75a507509686e8, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/3433444266e4457684a11e7f726601f4] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp, totalSize=48.4 K 2024-11-12T06:50:09,130 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142746 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:50:09,130 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): f8bcab44974ca57906f4d57bd4289b93/A is initiating minor compaction (all files) 2024-11-12T06:50:09,130 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8bcab44974ca57906f4d57bd4289b93/A in TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:09,130 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/2f3845f84dc4444796130684a3900109, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/a1a5dc9c667b41e392a33228a1b0258e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/899b0a43857b4d6b8dbf9a746c89e190, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/56416897c9a64a5584e9a9d47383ae7d] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp, totalSize=139.4 K 2024-11-12T06:50:09,130 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:09,130 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. files: [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/2f3845f84dc4444796130684a3900109, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/a1a5dc9c667b41e392a33228a1b0258e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/899b0a43857b4d6b8dbf9a746c89e190, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/56416897c9a64a5584e9a9d47383ae7d] 2024-11-12T06:50:09,130 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 757aff8019894c89aee6b9cbbe6066f4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731394202738 2024-11-12T06:50:09,130 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f3845f84dc4444796130684a3900109, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731394202738 2024-11-12T06:50:09,131 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1a5dc9c667b41e392a33228a1b0258e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731394204459 2024-11-12T06:50:09,131 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 1470e421fb304d6f81d429814ecbda56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731394204459 2024-11-12T06:50:09,131 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 899b0a43857b4d6b8dbf9a746c89e190, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731394205587 2024-11-12T06:50:09,131 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 03533005ff1041c9bb75a507509686e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731394205587 2024-11-12T06:50:09,131 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 3433444266e4457684a11e7f726601f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731394206710 2024-11-12T06:50:09,131 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56416897c9a64a5584e9a9d47383ae7d, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731394206706 2024-11-12T06:50:09,139 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8bcab44974ca57906f4d57bd4289b93#B#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:50:09,140 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/66917f8269e94639a8042ebbf5d24602 is 50, key is test_row_0/B:col10/1731394206710/Put/seqid=0 2024-11-12T06:50:09,155 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:09,159 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411121f08589c220145b9816d7126b1cb64c9_f8bcab44974ca57906f4d57bd4289b93 store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:09,161 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411121f08589c220145b9816d7126b1cb64c9_f8bcab44974ca57906f4d57bd4289b93, store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:09,161 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121f08589c220145b9816d7126b1cb64c9_f8bcab44974ca57906f4d57bd4289b93 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:09,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742394_1570 (size=4469) 2024-11-12T06:50:09,167 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8bcab44974ca57906f4d57bd4289b93#A#compaction#475 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:50:09,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742393_1569 (size=12779) 2024-11-12T06:50:09,167 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/d9346bffc4e24bb7953697902f56f07e is 175, key is test_row_0/A:col10/1731394206710/Put/seqid=0 2024-11-12T06:50:09,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742395_1571 (size=31733) 2024-11-12T06:50:09,172 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/66917f8269e94639a8042ebbf5d24602 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/66917f8269e94639a8042ebbf5d24602 2024-11-12T06:50:09,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T06:50:09,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:09,177 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8bcab44974ca57906f4d57bd4289b93/B of f8bcab44974ca57906f4d57bd4289b93 into 66917f8269e94639a8042ebbf5d24602(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:50:09,177 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing f8bcab44974ca57906f4d57bd4289b93 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-12T06:50:09,177 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:09,177 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., storeName=f8bcab44974ca57906f4d57bd4289b93/B, priority=12, startTime=1731394209128; duration=0sec 2024-11-12T06:50:09,177 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:50:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=A 2024-11-12T06:50:09,177 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8bcab44974ca57906f4d57bd4289b93:B 2024-11-12T06:50:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=B 2024-11-12T06:50:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:09,177 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:50:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=C 2024-11-12T06:50:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:09,178 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49546 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:50:09,178 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): f8bcab44974ca57906f4d57bd4289b93/C is initiating minor compaction (all files) 2024-11-12T06:50:09,178 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8bcab44974ca57906f4d57bd4289b93/C in TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:09,178 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/92afd40a2a274471a8f084648f4cc716, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/8a2a3ca4ab3b44d788faaf2045a21fdf, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/63af74117c5149b5ab266ef288b21800, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/a4f008844d4f4f3ea9e0368f33265d58] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp, totalSize=48.4 K 2024-11-12T06:50:09,179 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 92afd40a2a274471a8f084648f4cc716, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731394202738 2024-11-12T06:50:09,179 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a2a3ca4ab3b44d788faaf2045a21fdf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731394204459 2024-11-12T06:50:09,180 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 63af74117c5149b5ab266ef288b21800, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731394205587 2024-11-12T06:50:09,180 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting a4f008844d4f4f3ea9e0368f33265d58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731394206710 2024-11-12T06:50:09,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111248a39b9191334c9cbe32a23d77dd49ec_f8bcab44974ca57906f4d57bd4289b93 is 50, key is test_row_0/A:col10/1731394207879/Put/seqid=0 2024-11-12T06:50:09,188 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8bcab44974ca57906f4d57bd4289b93#C#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:50:09,188 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/96b68cc7f0d54dc684cf1a0aae989b33 is 50, key is test_row_0/C:col10/1731394206710/Put/seqid=0 2024-11-12T06:50:09,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742396_1572 (size=12454) 2024-11-12T06:50:09,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742397_1573 (size=12779) 2024-11-12T06:50:09,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,221 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111248a39b9191334c9cbe32a23d77dd49ec_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111248a39b9191334c9cbe32a23d77dd49ec_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:09,221 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/96b68cc7f0d54dc684cf1a0aae989b33 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/96b68cc7f0d54dc684cf1a0aae989b33 2024-11-12T06:50:09,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/641fd3beed9342df94389f730f8d5968, store: [table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:09,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/641fd3beed9342df94389f730f8d5968 is 175, key is test_row_0/A:col10/1731394207879/Put/seqid=0 2024-11-12T06:50:09,227 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8bcab44974ca57906f4d57bd4289b93/C of f8bcab44974ca57906f4d57bd4289b93 into 96b68cc7f0d54dc684cf1a0aae989b33(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:50:09,227 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:09,227 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., storeName=f8bcab44974ca57906f4d57bd4289b93/C, priority=12, startTime=1731394209128; duration=0sec 2024-11-12T06:50:09,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742398_1574 (size=31255) 2024-11-12T06:50:09,227 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:09,227 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8bcab44974ca57906f4d57bd4289b93:C 2024-11-12T06:50:09,250 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=462, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/641fd3beed9342df94389f730f8d5968 2024-11-12T06:50:09,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/96a25c1be3b94acc9ff5a277d1cea5c8 is 50, key is test_row_0/B:col10/1731394207879/Put/seqid=0 2024-11-12T06:50:09,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742399_1575 (size=12301) 2024-11-12T06:50:09,261 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/96a25c1be3b94acc9ff5a277d1cea5c8 2024-11-12T06:50:09,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/ac5088ec29a8498f8ca87c3886359d04 is 50, key is test_row_0/C:col10/1731394207879/Put/seqid=0 2024-11-12T06:50:09,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742400_1576 (size=12301) 2024-11-12T06:50:09,284 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/ac5088ec29a8498f8ca87c3886359d04 2024-11-12T06:50:09,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/641fd3beed9342df94389f730f8d5968 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/641fd3beed9342df94389f730f8d5968 2024-11-12T06:50:09,292 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/641fd3beed9342df94389f730f8d5968, entries=150, sequenceid=462, filesize=30.5 K 2024-11-12T06:50:09,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/96a25c1be3b94acc9ff5a277d1cea5c8 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/96a25c1be3b94acc9ff5a277d1cea5c8 2024-11-12T06:50:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,296 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/96a25c1be3b94acc9ff5a277d1cea5c8, entries=150, sequenceid=462, filesize=12.0 K 2024-11-12T06:50:09,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/ac5088ec29a8498f8ca87c3886359d04 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/ac5088ec29a8498f8ca87c3886359d04 2024-11-12T06:50:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,301 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/ac5088ec29a8498f8ca87c3886359d04, entries=150, sequenceid=462, filesize=12.0 K 2024-11-12T06:50:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,302 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for f8bcab44974ca57906f4d57bd4289b93 in 125ms, sequenceid=462, compaction requested=false 2024-11-12T06:50:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:09,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:09,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-12T06:50:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-12T06:50:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-12T06:50:09,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0440 sec 2024-11-12T06:50:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,307 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.0480 sec 2024-11-12T06:50:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T06:50:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,397 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:50:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,397 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:50:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-12T06:50:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,399 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:50:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T06:50:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,400 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:50:09,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:50:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T06:50:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T06:50:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:09,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:09,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:09,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-12T06:50:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-12T06:50:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-12T06:50:09,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-12T06:50:09,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 159 msec 2024-11-12T06:50:09,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,576 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/d9346bffc4e24bb7953697902f56f07e as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/d9346bffc4e24bb7953697902f56f07e 2024-11-12T06:50:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,580 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8bcab44974ca57906f4d57bd4289b93/A of f8bcab44974ca57906f4d57bd4289b93 into d9346bffc4e24bb7953697902f56f07e(size=31.0 K), total size for store is 61.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:50:09,580 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:09,580 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., storeName=f8bcab44974ca57906f4d57bd4289b93/A, priority=12, startTime=1731394209128; duration=0sec 2024-11-12T06:50:09,580 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,580 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8bcab44974ca57906f4d57bd4289b93:A 2024-11-12T06:50:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T06:50:09,717 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:50:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,718 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:50:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-12T06:50:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,719 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:50:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T06:50:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,720 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:50:09,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:50:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T06:50:09,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8bcab44974ca57906f4d57bd4289b93 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T06:50:09,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=A 2024-11-12T06:50:09,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:09,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=B 2024-11-12T06:50:09,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:09,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=C 2024-11-12T06:50:09,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:09,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:09,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123fa0b0e3389847e4844b7a3472d34d75_f8bcab44974ca57906f4d57bd4289b93 is 50, key is test_row_0/A:col10/1731394209845/Put/seqid=0 2024-11-12T06:50:09,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T06:50:09,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:09,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:09,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:09,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:09,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:09,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742401_1577 (size=12454) 2024-11-12T06:50:09,883 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:09,887 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123fa0b0e3389847e4844b7a3472d34d75_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123fa0b0e3389847e4844b7a3472d34d75_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:09,888 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/634c2bf4aac64ddc9f866916d286c3ff, store: [table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:09,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/634c2bf4aac64ddc9f866916d286c3ff is 175, key is test_row_0/A:col10/1731394209845/Put/seqid=0 2024-11-12T06:50:09,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742402_1578 (size=31255) 2024-11-12T06:50:09,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394209928, max timestamp=1731394209962Num uniques 859; 2024-11-12T06:50:09,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:50:09,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394209928, max timestamp=1731394209962Num uniques 859; 2024-11-12T06:50:09,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:50:09,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731394209928, max timestamp=1731394209962Num uniques 859; 2024-11-12T06:50:09,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T06:50:10,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T06:50:10,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:10,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:10,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:10,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:10,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T06:50:10,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731394210050, max timestamp=1731394210063Num uniques 1018; 2024-11-12T06:50:10,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-12T06:50:10,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731394210050, max timestamp=1731394210063Num uniques 1018; 2024-11-12T06:50:10,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-12T06:50:10,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731394210050, max timestamp=1731394210063Num uniques 1018; 2024-11-12T06:50:10,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-12T06:50:10,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:10,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394270097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:10,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:10,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394270104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:10,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394270105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:10,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:10,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T06:50:10,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:10,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:10,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:10,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:10,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:10,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:10,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394270207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:10,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:10,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394270227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:10,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:10,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394270237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:10,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:10,309 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=475, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/634c2bf4aac64ddc9f866916d286c3ff 2024-11-12T06:50:10,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/856d98a274f04cc087f9d3bfda54316a is 50, key is test_row_0/B:col10/1731394209845/Put/seqid=0 2024-11-12T06:50:10,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742403_1579 (size=12301) 2024-11-12T06:50:10,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/856d98a274f04cc087f9d3bfda54316a 2024-11-12T06:50:10,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T06:50:10,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:10,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:10,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:10,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:10,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:10,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/bc7e5d35a6634b74829b1e6833008ea2 is 50, key is test_row_0/C:col10/1731394209845/Put/seqid=0 2024-11-12T06:50:10,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742404_1580 (size=12301) 2024-11-12T06:50:10,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/bc7e5d35a6634b74829b1e6833008ea2 2024-11-12T06:50:10,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/634c2bf4aac64ddc9f866916d286c3ff as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/634c2bf4aac64ddc9f866916d286c3ff 2024-11-12T06:50:10,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T06:50:10,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/634c2bf4aac64ddc9f866916d286c3ff, entries=150, sequenceid=475, filesize=30.5 K 2024-11-12T06:50:10,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/856d98a274f04cc087f9d3bfda54316a as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/856d98a274f04cc087f9d3bfda54316a 2024-11-12T06:50:10,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/856d98a274f04cc087f9d3bfda54316a, entries=150, sequenceid=475, filesize=12.0 K 2024-11-12T06:50:10,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/bc7e5d35a6634b74829b1e6833008ea2 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/bc7e5d35a6634b74829b1e6833008ea2 2024-11-12T06:50:10,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/bc7e5d35a6634b74829b1e6833008ea2, entries=150, sequenceid=475, filesize=12.0 K 2024-11-12T06:50:10,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f8bcab44974ca57906f4d57bd4289b93 in 509ms, sequenceid=475, compaction requested=true 2024-11-12T06:50:10,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:10,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8bcab44974ca57906f4d57bd4289b93:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:50:10,356 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:50:10,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:10,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8bcab44974ca57906f4d57bd4289b93:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:50:10,356 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:50:10,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:10,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8bcab44974ca57906f4d57bd4289b93:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:50:10,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:50:10,357 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:50:10,357 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:50:10,357 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): f8bcab44974ca57906f4d57bd4289b93/A is initiating minor compaction (all files) 2024-11-12T06:50:10,357 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): f8bcab44974ca57906f4d57bd4289b93/B is initiating minor compaction (all files) 2024-11-12T06:50:10,357 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8bcab44974ca57906f4d57bd4289b93/A in TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:10,357 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/d9346bffc4e24bb7953697902f56f07e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/641fd3beed9342df94389f730f8d5968, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/634c2bf4aac64ddc9f866916d286c3ff] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp, totalSize=92.0 K 2024-11-12T06:50:10,357 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8bcab44974ca57906f4d57bd4289b93/B in TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:10,357 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:10,357 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. files: [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/d9346bffc4e24bb7953697902f56f07e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/641fd3beed9342df94389f730f8d5968, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/634c2bf4aac64ddc9f866916d286c3ff] 2024-11-12T06:50:10,357 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/66917f8269e94639a8042ebbf5d24602, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/96a25c1be3b94acc9ff5a277d1cea5c8, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/856d98a274f04cc087f9d3bfda54316a] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp, totalSize=36.5 K 2024-11-12T06:50:10,358 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 66917f8269e94639a8042ebbf5d24602, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731394206710 2024-11-12T06:50:10,358 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9346bffc4e24bb7953697902f56f07e, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731394206710 2024-11-12T06:50:10,359 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 641fd3beed9342df94389f730f8d5968, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731394207868 2024-11-12T06:50:10,359 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 96a25c1be3b94acc9ff5a277d1cea5c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731394207868 2024-11-12T06:50:10,359 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 634c2bf4aac64ddc9f866916d286c3ff, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731394209801 2024-11-12T06:50:10,359 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 856d98a274f04cc087f9d3bfda54316a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731394209801 2024-11-12T06:50:10,363 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:10,365 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8bcab44974ca57906f4d57bd4289b93#B#compaction#484 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:50:10,365 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/24a490e1bb394a4c9cf2e77a1f671068 is 50, key is test_row_0/B:col10/1731394209845/Put/seqid=0 2024-11-12T06:50:10,365 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111261416c4667874e61aeb26330c1cedb7b_f8bcab44974ca57906f4d57bd4289b93 store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:10,368 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111261416c4667874e61aeb26330c1cedb7b_f8bcab44974ca57906f4d57bd4289b93, store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:10,368 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111261416c4667874e61aeb26330c1cedb7b_f8bcab44974ca57906f4d57bd4289b93 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:10,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742405_1581 (size=12881) 2024-11-12T06:50:10,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742406_1582 (size=4469) 2024-11-12T06:50:10,374 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8bcab44974ca57906f4d57bd4289b93#A#compaction#483 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:50:10,375 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/49fda5a6a24a4425a7d8b185fbd16644 is 175, key is test_row_0/A:col10/1731394209845/Put/seqid=0 2024-11-12T06:50:10,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742407_1583 (size=31835) 2024-11-12T06:50:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:10,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8bcab44974ca57906f4d57bd4289b93 3/3 column families, dataSize=140.89 KB heapSize=394.69 KB 2024-11-12T06:50:10,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=A 2024-11-12T06:50:10,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:50:10,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=B 2024-11-12T06:50:10,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:50:10,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=C 2024-11-12T06:50:10,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:50:10,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111226a96be089fb44c1b5a9538f72a21a60_f8bcab44974ca57906f4d57bd4289b93 is 50, key is test_row_0/A:col10/1731394210102/Put/seqid=0 2024-11-12T06:50:10,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742408_1584 (size=14994) 2024-11-12T06:50:10,434 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:10,440 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111226a96be089fb44c1b5a9538f72a21a60_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111226a96be089fb44c1b5a9538f72a21a60_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:10,440 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/71829f68a2b24197a3a3f30ce6eebd9a, store: [table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:10,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/71829f68a2b24197a3a3f30ce6eebd9a is 175, key is test_row_0/A:col10/1731394210102/Put/seqid=0 2024-11-12T06:50:10,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742409_1585 (size=39949) 2024-11-12T06:50:10,445 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=528, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/71829f68a2b24197a3a3f30ce6eebd9a 2024-11-12T06:50:10,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:10,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394270444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:10,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:10,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394270447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:10,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:10,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/23c92fe9e693457a9cfd8fdfe9ae42f1 is 50, key is test_row_0/B:col10/1731394210102/Put/seqid=0 2024-11-12T06:50:10,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394270449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:10,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742410_1586 (size=12301) 2024-11-12T06:50:10,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T06:50:10,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:10,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:10,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:10,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:10,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:10,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:10,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394270557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:10,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394270557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:10,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:10,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:10,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T06:50:10,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:10,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:10,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:10,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:10,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:10,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:10,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394270757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:10,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:10,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394270767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394270767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:10,773 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/24a490e1bb394a4c9cf2e77a1f671068 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/24a490e1bb394a4c9cf2e77a1f671068 2024-11-12T06:50:10,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:10,776 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8bcab44974ca57906f4d57bd4289b93/B of f8bcab44974ca57906f4d57bd4289b93 into 24a490e1bb394a4c9cf2e77a1f671068(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:50:10,776 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:10,776 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., storeName=f8bcab44974ca57906f4d57bd4289b93/B, priority=13, startTime=1731394210356; duration=0sec 2024-11-12T06:50:10,776 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:50:10,776 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8bcab44974ca57906f4d57bd4289b93:B 2024-11-12T06:50:10,776 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T06:50:10,777 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T06:50:10,777 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): f8bcab44974ca57906f4d57bd4289b93/C is initiating minor compaction (all files) 2024-11-12T06:50:10,777 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8bcab44974ca57906f4d57bd4289b93/C in TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:10,777 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/96b68cc7f0d54dc684cf1a0aae989b33, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/ac5088ec29a8498f8ca87c3886359d04, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/bc7e5d35a6634b74829b1e6833008ea2] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp, totalSize=36.5 K 2024-11-12T06:50:10,777 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 96b68cc7f0d54dc684cf1a0aae989b33, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731394206710 2024-11-12T06:50:10,777 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting ac5088ec29a8498f8ca87c3886359d04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731394207868 2024-11-12T06:50:10,778 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting bc7e5d35a6634b74829b1e6833008ea2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731394209801 2024-11-12T06:50:10,783 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/49fda5a6a24a4425a7d8b185fbd16644 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/49fda5a6a24a4425a7d8b185fbd16644 2024-11-12T06:50:10,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T06:50:10,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:10,786 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8bcab44974ca57906f4d57bd4289b93#C#compaction#487 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:50:10,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:10,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:10,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:10,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:10,787 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/d9c25187dfbb42d0bde24e4314009076 is 50, key is test_row_0/C:col10/1731394209845/Put/seqid=0 2024-11-12T06:50:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:10,790 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8bcab44974ca57906f4d57bd4289b93/A of f8bcab44974ca57906f4d57bd4289b93 into 49fda5a6a24a4425a7d8b185fbd16644(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:50:10,790 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:10,790 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., storeName=f8bcab44974ca57906f4d57bd4289b93/A, priority=13, startTime=1731394210356; duration=0sec 2024-11-12T06:50:10,791 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:10,791 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8bcab44974ca57906f4d57bd4289b93:A 2024-11-12T06:50:10,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742411_1587 (size=12881) 2024-11-12T06:50:10,801 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/d9c25187dfbb42d0bde24e4314009076 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/d9c25187dfbb42d0bde24e4314009076 2024-11-12T06:50:10,805 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8bcab44974ca57906f4d57bd4289b93/C of f8bcab44974ca57906f4d57bd4289b93 into d9c25187dfbb42d0bde24e4314009076(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:50:10,805 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:10,805 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., storeName=f8bcab44974ca57906f4d57bd4289b93/C, priority=13, startTime=1731394210356; duration=0sec 2024-11-12T06:50:10,805 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:10,805 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8bcab44974ca57906f4d57bd4289b93:C 2024-11-12T06:50:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T06:50:10,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/23c92fe9e693457a9cfd8fdfe9ae42f1 2024-11-12T06:50:10,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/78f17e172a56426b9f5945473228c678 is 50, key is test_row_0/C:col10/1731394210102/Put/seqid=0 2024-11-12T06:50:10,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742412_1588 (size=12301) 2024-11-12T06:50:10,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T06:50:10,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:10,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:10,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:10,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:10,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:10,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55690 deadline: 1731394270967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:10,971 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_0' in f8bcab44974ca57906f4d57bd4289b93 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:50:10,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:10,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:11,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394271077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:11,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:11,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:11,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394271077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:11,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:11,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:11,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:11,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:11,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T06:50:11,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:11,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:11,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:11,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:11,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:11,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T06:50:11,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:11,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:11,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:11,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:11,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:11,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/78f17e172a56426b9f5945473228c678 2024-11-12T06:50:11,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394271267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:11,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/71829f68a2b24197a3a3f30ce6eebd9a as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/71829f68a2b24197a3a3f30ce6eebd9a 2024-11-12T06:50:11,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/71829f68a2b24197a3a3f30ce6eebd9a, entries=200, sequenceid=528, filesize=39.0 K 2024-11-12T06:50:11,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/23c92fe9e693457a9cfd8fdfe9ae42f1 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/23c92fe9e693457a9cfd8fdfe9ae42f1 2024-11-12T06:50:11,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/23c92fe9e693457a9cfd8fdfe9ae42f1, entries=150, sequenceid=528, filesize=12.0 K 2024-11-12T06:50:11,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/78f17e172a56426b9f5945473228c678 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/78f17e172a56426b9f5945473228c678 2024-11-12T06:50:11,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/78f17e172a56426b9f5945473228c678, entries=150, sequenceid=528, filesize=12.0 K 2024-11-12T06:50:11,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~394.64 KB/404112, currentSize=40.25 KB/41220 for f8bcab44974ca57906f4d57bd4289b93 in 858ms, sequenceid=528, compaction requested=false 2024-11-12T06:50:11,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:11,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T06:50:11,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:11,396 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing f8bcab44974ca57906f4d57bd4289b93 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-12T06:50:11,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=A 2024-11-12T06:50:11,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:11,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=B 2024-11-12T06:50:11,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:11,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=C 2024-11-12T06:50:11,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:11,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120b0a8b3240a9468199fd1f1fcf11fe5a_f8bcab44974ca57906f4d57bd4289b93 is 50, key is test_row_0/A:col10/1731394210440/Put/seqid=0 2024-11-12T06:50:11,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742413_1589 (size=12454) 2024-11-12T06:50:11,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:11,410 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120b0a8b3240a9468199fd1f1fcf11fe5a_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120b0a8b3240a9468199fd1f1fcf11fe5a_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:11,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/395d6ad98f1f46a8a62ed9c7fbce8961, store: [table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:11,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/395d6ad98f1f46a8a62ed9c7fbce8961 is 175, key is test_row_0/A:col10/1731394210440/Put/seqid=0 2024-11-12T06:50:11,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742414_1590 (size=31255) 2024-11-12T06:50:11,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:11,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:11,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394211598, max timestamp=1731394211622Num uniques 859; 2024-11-12T06:50:11,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:50:11,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394211598, max timestamp=1731394211622Num uniques 859; 2024-11-12T06:50:11,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:50:11,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394211598, max timestamp=1731394211622Num uniques 859; 2024-11-12T06:50:11,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:50:11,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:11,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394271662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:11,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:11,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:11,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:11,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:11,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394271665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:11,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:11,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:11,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:11,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394271777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:11,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394271777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:11,815 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=540, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/395d6ad98f1f46a8a62ed9c7fbce8961 2024-11-12T06:50:11,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/cfb8bc3f327f47dda9c0870de5b37a1d is 50, key is test_row_0/B:col10/1731394210440/Put/seqid=0 2024-11-12T06:50:11,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742415_1591 (size=12301) 2024-11-12T06:50:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T06:50:11,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394271987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:11,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:11,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394271987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:11,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:11,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:11,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:11,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:11,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:12,223 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/cfb8bc3f327f47dda9c0870de5b37a1d 2024-11-12T06:50:12,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/8cf6e9c3a4204a19829d30f8eec8929b is 50, key is test_row_0/C:col10/1731394210440/Put/seqid=0 2024-11-12T06:50:12,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742416_1592 (size=12301) 2024-11-12T06:50:12,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394272287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:12,290 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_0' in f8bcab44974ca57906f4d57bd4289b93 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:50:12,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:12,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:12,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:12,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394272297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:12,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394272297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:12,632 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/8cf6e9c3a4204a19829d30f8eec8929b 2024-11-12T06:50:12,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/395d6ad98f1f46a8a62ed9c7fbce8961 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/395d6ad98f1f46a8a62ed9c7fbce8961 2024-11-12T06:50:12,638 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/395d6ad98f1f46a8a62ed9c7fbce8961, entries=150, sequenceid=540, filesize=30.5 K 2024-11-12T06:50:12,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/cfb8bc3f327f47dda9c0870de5b37a1d as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/cfb8bc3f327f47dda9c0870de5b37a1d 2024-11-12T06:50:12,642 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/cfb8bc3f327f47dda9c0870de5b37a1d, entries=150, sequenceid=540, filesize=12.0 K 2024-11-12T06:50:12,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/8cf6e9c3a4204a19829d30f8eec8929b as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/8cf6e9c3a4204a19829d30f8eec8929b 2024-11-12T06:50:12,645 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/8cf6e9c3a4204a19829d30f8eec8929b, entries=150, sequenceid=540, filesize=12.0 K 2024-11-12T06:50:12,646 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for f8bcab44974ca57906f4d57bd4289b93 in 1250ms, sequenceid=540, compaction requested=true 2024-11-12T06:50:12,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:12,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:12,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-12T06:50:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-12T06:50:12,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-12T06:50:12,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9270 sec 2024-11-12T06:50:12,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.9310 sec 2024-11-12T06:50:12,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:12,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731394211651, max timestamp=1731394212810Num uniques 1068; 2024-11-12T06:50:12,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731394211651, max timestamp=1731394212810Num uniques 1068; 2024-11-12T06:50:12,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-12T06:50:12,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731394211651, max timestamp=1731394212810Num uniques 1068; 2024-11-12T06:50:12,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-12T06:50:12,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-12T06:50:12,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8bcab44974ca57906f4d57bd4289b93 3/3 column families, dataSize=33.54 KB heapSize=85.31 KB 2024-11-12T06:50:12,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=A 2024-11-12T06:50:12,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:50:12,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=B 2024-11-12T06:50:12,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:50:12,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=C 2024-11-12T06:50:12,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:50:12,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a17b7e3e75de4f86a3b421aaf048f69a_f8bcab44974ca57906f4d57bd4289b93 is 50, key is test_row_0/A:col10/1731394211664/Put/seqid=0 2024-11-12T06:50:12,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742417_1593 (size=12454) 2024-11-12T06:50:12,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394212867, max timestamp=1731394212879Num uniques 859; 2024-11-12T06:50:12,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:50:12,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394212867, max timestamp=1731394212879Num uniques 859; 2024-11-12T06:50:12,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:50:12,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731394212867, max timestamp=1731394212879Num uniques 859; 2024-11-12T06:50:12,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T06:50:12,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731394212919, max timestamp=1731394212931Num uniques 1068; 2024-11-12T06:50:12,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T06:50:12,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731394212919, max timestamp=1731394212931Num uniques 1068; 2024-11-12T06:50:12,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T06:50:12,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731394212919, max timestamp=1731394212931Num uniques 1068; 2024-11-12T06:50:12,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T06:50:12,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731394212962, max timestamp=1731394212977Num uniques 1077; 2024-11-12T06:50:12,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731394212962, max timestamp=1731394212977Num uniques 1077; 2024-11-12T06:50:12,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-12T06:50:12,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-12T06:50:12,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731394212962, max timestamp=1731394212977Num uniques 1077; 2024-11-12T06:50:12,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-12T06:50:13,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731394213011, max timestamp=1731394213047Num uniques 1086; 2024-11-12T06:50:13,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-12T06:50:13,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731394213011, max timestamp=1731394213047Num uniques 1086; 2024-11-12T06:50:13,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-12T06:50:13,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731394213011, max timestamp=1731394213047Num uniques 1086; 2024-11-12T06:50:13,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-12T06:50:13,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731394213085, max timestamp=1731394213128Num uniques 1045; 2024-11-12T06:50:13,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-12T06:50:13,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731394213105, max timestamp=1731394213128Num uniques 1045; 2024-11-12T06:50:13,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-12T06:50:13,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731394213105, max timestamp=1731394213128Num uniques 1045; 2024-11-12T06:50:13,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-12T06:50:13,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731394213182, max timestamp=1731394213207Num uniques 1054; 2024-11-12T06:50:13,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-12T06:50:13,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731394213182, max timestamp=1731394213207Num uniques 1054; 2024-11-12T06:50:13,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-12T06:50:13,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731394213182, max timestamp=1731394213207Num uniques 1104; 2024-11-12T06:50:13,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-12T06:50:13,232 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:13,236 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a17b7e3e75de4f86a3b421aaf048f69a_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a17b7e3e75de4f86a3b421aaf048f69a_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:13,237 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/f771315f2da34c2fa85db1347f8a2a3a, store: [table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:13,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/f771315f2da34c2fa85db1347f8a2a3a is 175, key is test_row_0/A:col10/1731394211664/Put/seqid=0 2024-11-12T06:50:13,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742418_1594 (size=31255) 2024-11-12T06:50:13,260 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=579, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/f771315f2da34c2fa85db1347f8a2a3a 2024-11-12T06:50:13,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/d39db72369ec4ad3a7fafec5b787f29f is 50, key is test_row_0/B:col10/1731394211664/Put/seqid=0 2024-11-12T06:50:13,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731394213260, max timestamp=1731394213282Num uniques 1063; 2024-11-12T06:50:13,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-12T06:50:13,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731394213260, max timestamp=1731394213282Num uniques 1063; 2024-11-12T06:50:13,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731394213260, max timestamp=1731394213282Num uniques 1063; 2024-11-12T06:50:13,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-12T06:50:13,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-12T06:50:13,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742419_1595 (size=12301) 2024-11-12T06:50:13,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731394213326, max timestamp=1731394213349Num uniques 1022; 2024-11-12T06:50:13,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-12T06:50:13,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731394213335, max timestamp=1731394213349Num uniques 1022; 2024-11-12T06:50:13,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=6870, suffix heap size=134256,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=150 2024-11-12T06:50:13,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731394213335, max timestamp=1731394213349Num uniques 1022; 2024-11-12T06:50:13,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=6870, suffix heap size=134256,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=150 2024-11-12T06:50:13,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731394213395, max timestamp=1731394213414Num uniques 1009; 2024-11-12T06:50:13,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10576, suffix heap size=132904,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=231 2024-11-12T06:50:13,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731394213395, max timestamp=1731394213414Num uniques 1081; 2024-11-12T06:50:13,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-12T06:50:13,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731394213395, max timestamp=1731394213414Num uniques 1009; 2024-11-12T06:50:13,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10576, suffix heap size=132904,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=231 2024-11-12T06:50:13,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731394213447, max timestamp=1731394213468Num uniques 1090; 2024-11-12T06:50:13,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-12T06:50:13,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731394213447, max timestamp=1731394213468Num uniques 1090; 2024-11-12T06:50:13,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-12T06:50:13,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731394213447, max timestamp=1731394213468Num uniques 1090; 2024-11-12T06:50:13,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-12T06:50:13,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731394213502, max timestamp=1731394213534Num uniques 1099; 2024-11-12T06:50:13,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=9114, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-11-12T06:50:13,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731394213502, max timestamp=1731394213534Num uniques 1099; 2024-11-12T06:50:13,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=9114, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-11-12T06:50:13,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731394213502, max timestamp=1731394213534Num uniques 1099; 2024-11-12T06:50:13,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=9114, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-11-12T06:50:13,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731394213569, max timestamp=1731394213624Num uniques 1058; 2024-11-12T06:50:13,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-12T06:50:13,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731394213569, max timestamp=1731394213624Num uniques 1058; 2024-11-12T06:50:13,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-12T06:50:13,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731394213569, max timestamp=1731394213624Num uniques 1058; 2024-11-12T06:50:13,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-12T06:50:13,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731394213663, max timestamp=1731394213684Num uniques 1017; 2024-11-12T06:50:13,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-11-12T06:50:13,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731394213663, max timestamp=1731394213684Num uniques 1017; 2024-11-12T06:50:13,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-11-12T06:50:13,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731394213663, max timestamp=1731394213684Num uniques 1017; 2024-11-12T06:50:13,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9f0beee3d6b6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-11-12T06:50:13,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/d39db72369ec4ad3a7fafec5b787f29f 2024-11-12T06:50:13,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/0c2b09ecf1504589a799640b12afa1f6 is 50, key is test_row_0/C:col10/1731394211664/Put/seqid=0 2024-11-12T06:50:13,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742420_1596 (size=12301) 2024-11-12T06:50:13,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/0c2b09ecf1504589a799640b12afa1f6 2024-11-12T06:50:13,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/f771315f2da34c2fa85db1347f8a2a3a as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/f771315f2da34c2fa85db1347f8a2a3a 2024-11-12T06:50:13,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/f771315f2da34c2fa85db1347f8a2a3a, entries=150, sequenceid=579, filesize=30.5 K 2024-11-12T06:50:13,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/d39db72369ec4ad3a7fafec5b787f29f as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/d39db72369ec4ad3a7fafec5b787f29f 2024-11-12T06:50:13,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/d39db72369ec4ad3a7fafec5b787f29f, entries=150, sequenceid=579, filesize=12.0 K 2024-11-12T06:50:13,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/0c2b09ecf1504589a799640b12afa1f6 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/0c2b09ecf1504589a799640b12afa1f6 2024-11-12T06:50:13,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/0c2b09ecf1504589a799640b12afa1f6, entries=150, sequenceid=579, filesize=12.0 K 2024-11-12T06:50:13,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=100.63 KB/103050 for f8bcab44974ca57906f4d57bd4289b93 in 905ms, sequenceid=579, compaction requested=true 2024-11-12T06:50:13,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:13,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8bcab44974ca57906f4d57bd4289b93:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:50:13,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:13,721 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:50:13,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8bcab44974ca57906f4d57bd4289b93:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:50:13,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:13,721 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:50:13,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8bcab44974ca57906f4d57bd4289b93:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:50:13,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:50:13,721 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:50:13,721 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:50:13,722 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): f8bcab44974ca57906f4d57bd4289b93/B is initiating minor compaction (all files) 2024-11-12T06:50:13,722 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): f8bcab44974ca57906f4d57bd4289b93/A is initiating minor compaction (all files) 2024-11-12T06:50:13,722 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8bcab44974ca57906f4d57bd4289b93/B in TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:13,722 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8bcab44974ca57906f4d57bd4289b93/A in TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:13,722 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/24a490e1bb394a4c9cf2e77a1f671068, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/23c92fe9e693457a9cfd8fdfe9ae42f1, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/cfb8bc3f327f47dda9c0870de5b37a1d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/d39db72369ec4ad3a7fafec5b787f29f] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp, totalSize=48.6 K 2024-11-12T06:50:13,722 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/49fda5a6a24a4425a7d8b185fbd16644, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/71829f68a2b24197a3a3f30ce6eebd9a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/395d6ad98f1f46a8a62ed9c7fbce8961, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/f771315f2da34c2fa85db1347f8a2a3a] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp, totalSize=131.1 K 2024-11-12T06:50:13,722 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:13,722 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. files: [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/49fda5a6a24a4425a7d8b185fbd16644, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/71829f68a2b24197a3a3f30ce6eebd9a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/395d6ad98f1f46a8a62ed9c7fbce8961, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/f771315f2da34c2fa85db1347f8a2a3a] 2024-11-12T06:50:13,722 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 24a490e1bb394a4c9cf2e77a1f671068, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731394209801 2024-11-12T06:50:13,722 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49fda5a6a24a4425a7d8b185fbd16644, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731394209801 2024-11-12T06:50:13,723 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 23c92fe9e693457a9cfd8fdfe9ae42f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=528, earliestPutTs=1731394210096 2024-11-12T06:50:13,723 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71829f68a2b24197a3a3f30ce6eebd9a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=528, earliestPutTs=1731394210096 2024-11-12T06:50:13,723 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting cfb8bc3f327f47dda9c0870de5b37a1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1731394210440 2024-11-12T06:50:13,723 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 395d6ad98f1f46a8a62ed9c7fbce8961, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1731394210440 2024-11-12T06:50:13,723 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting d39db72369ec4ad3a7fafec5b787f29f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1731394211664 2024-11-12T06:50:13,723 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting f771315f2da34c2fa85db1347f8a2a3a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1731394211664 2024-11-12T06:50:13,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:13,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8bcab44974ca57906f4d57bd4289b93 3/3 column families, dataSize=107.34 KB heapSize=304.55 KB 2024-11-12T06:50:13,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=A 2024-11-12T06:50:13,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:50:13,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=B 2024-11-12T06:50:13,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:50:13,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=C 2024-11-12T06:50:13,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T06:50:13,729 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:13,731 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8bcab44974ca57906f4d57bd4289b93#B#compaction#495 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:50:13,731 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/a7bc1e5cb158451a907f5012e0f66903 is 50, key is test_row_0/B:col10/1731394211664/Put/seqid=0 2024-11-12T06:50:13,733 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111203228225918543818a4a1fed20573fda_f8bcab44974ca57906f4d57bd4289b93 store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:13,735 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111203228225918543818a4a1fed20573fda_f8bcab44974ca57906f4d57bd4289b93, store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:13,735 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111203228225918543818a4a1fed20573fda_f8bcab44974ca57906f4d57bd4289b93 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:13,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742421_1597 (size=13017) 2024-11-12T06:50:13,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112689cbbf8d1d04232965c867640d10f94_f8bcab44974ca57906f4d57bd4289b93 is 50, key is test_row_0/A:col10/1731394213723/Put/seqid=0 2024-11-12T06:50:13,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742422_1598 (size=4469) 2024-11-12T06:50:13,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742423_1599 (size=14994) 2024-11-12T06:50:13,745 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:13,746 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8bcab44974ca57906f4d57bd4289b93#A#compaction#496 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:50:13,746 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/40965b6d31974cab88f05fedd4fb64e7 is 175, key is test_row_0/A:col10/1731394211664/Put/seqid=0 2024-11-12T06:50:13,749 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112689cbbf8d1d04232965c867640d10f94_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112689cbbf8d1d04232965c867640d10f94_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:13,749 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/f13b275afb4141e6a65844224d113dfc, store: [table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:13,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/f13b275afb4141e6a65844224d113dfc is 175, key is test_row_0/A:col10/1731394213723/Put/seqid=0 2024-11-12T06:50:13,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742424_1600 (size=31971) 2024-11-12T06:50:13,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742425_1601 (size=39949) 2024-11-12T06:50:13,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:13,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 305 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394273776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:13,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:13,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 309 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394273776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:13,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T06:50:13,888 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:50:13,888 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:50:13,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-12T06:50:13,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T06:50:13,889 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:50:13,890 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:50:13,890 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:50:13,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:13,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 306 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394273888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:13,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:13,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:13,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:13,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394273888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:13,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:13,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:13,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:13,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:13,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T06:50:14,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-12T06:50:14,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:14,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:14,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:14,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:14,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:14,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 307 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394274097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:14,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 311 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394274097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:14,142 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/a7bc1e5cb158451a907f5012e0f66903 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/a7bc1e5cb158451a907f5012e0f66903 2024-11-12T06:50:14,144 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8bcab44974ca57906f4d57bd4289b93/B of f8bcab44974ca57906f4d57bd4289b93 into a7bc1e5cb158451a907f5012e0f66903(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:50:14,145 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:14,145 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., storeName=f8bcab44974ca57906f4d57bd4289b93/B, priority=12, startTime=1731394213721; duration=0sec 2024-11-12T06:50:14,145 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:50:14,145 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8bcab44974ca57906f4d57bd4289b93:B 2024-11-12T06:50:14,145 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:50:14,145 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:50:14,146 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): f8bcab44974ca57906f4d57bd4289b93/C is initiating minor compaction (all files) 2024-11-12T06:50:14,146 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8bcab44974ca57906f4d57bd4289b93/C in TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:14,146 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/d9c25187dfbb42d0bde24e4314009076, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/78f17e172a56426b9f5945473228c678, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/8cf6e9c3a4204a19829d30f8eec8929b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/0c2b09ecf1504589a799640b12afa1f6] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp, totalSize=48.6 K 2024-11-12T06:50:14,146 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting d9c25187dfbb42d0bde24e4314009076, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731394209801 2024-11-12T06:50:14,146 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 78f17e172a56426b9f5945473228c678, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=528, earliestPutTs=1731394210096 2024-11-12T06:50:14,146 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cf6e9c3a4204a19829d30f8eec8929b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1731394210440 2024-11-12T06:50:14,146 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c2b09ecf1504589a799640b12afa1f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1731394211664 2024-11-12T06:50:14,152 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8bcab44974ca57906f4d57bd4289b93#C#compaction#498 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:50:14,153 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/40965b6d31974cab88f05fedd4fb64e7 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/40965b6d31974cab88f05fedd4fb64e7 2024-11-12T06:50:14,153 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=818, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/f13b275afb4141e6a65844224d113dfc 2024-11-12T06:50:14,157 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8bcab44974ca57906f4d57bd4289b93/A of f8bcab44974ca57906f4d57bd4289b93 into 40965b6d31974cab88f05fedd4fb64e7(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:50:14,157 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:14,157 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., storeName=f8bcab44974ca57906f4d57bd4289b93/A, priority=12, startTime=1731394213720; duration=0sec 2024-11-12T06:50:14,157 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:14,157 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8bcab44974ca57906f4d57bd4289b93:A 2024-11-12T06:50:14,158 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/75b392d35b5745818212c058cd2597d5 is 50, key is test_row_0/C:col10/1731394211664/Put/seqid=0 2024-11-12T06:50:14,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/17cf292a5c2a483eb8a9c71b75018099 is 50, key is test_row_0/B:col10/1731394213723/Put/seqid=0 2024-11-12T06:50:14,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742426_1602 (size=13017) 2024-11-12T06:50:14,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742427_1603 (size=12301) 2024-11-12T06:50:14,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=818 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/17cf292a5c2a483eb8a9c71b75018099 2024-11-12T06:50:14,170 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/75b392d35b5745818212c058cd2597d5 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/75b392d35b5745818212c058cd2597d5 2024-11-12T06:50:14,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/afd93e3d231f451b93e17505e63266ba is 50, key is test_row_0/C:col10/1731394213723/Put/seqid=0 2024-11-12T06:50:14,174 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8bcab44974ca57906f4d57bd4289b93/C of f8bcab44974ca57906f4d57bd4289b93 into 75b392d35b5745818212c058cd2597d5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:50:14,174 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:14,174 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., storeName=f8bcab44974ca57906f4d57bd4289b93/C, priority=12, startTime=1731394213721; duration=0sec 2024-11-12T06:50:14,174 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:14,174 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8bcab44974ca57906f4d57bd4289b93:C 2024-11-12T06:50:14,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742428_1604 (size=12301) 2024-11-12T06:50:14,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-12T06:50:14,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:14,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:14,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:14,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:14,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T06:50:14,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55630 deadline: 1731394274297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:14,300 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_0' in f8bcab44974ca57906f4d57bd4289b93 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:50:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:14,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-12T06:50:14,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:14,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:14,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:14,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:14,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:14,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:14,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 308 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394274417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 312 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394274417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:14,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:14,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:14,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:14,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:14,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:14,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:14,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-12T06:50:14,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T06:50:14,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:14,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:14,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:14,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:14,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=818 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/afd93e3d231f451b93e17505e63266ba 2024-11-12T06:50:14,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/f13b275afb4141e6a65844224d113dfc as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/f13b275afb4141e6a65844224d113dfc 2024-11-12T06:50:14,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/f13b275afb4141e6a65844224d113dfc, entries=200, sequenceid=818, filesize=39.0 K 2024-11-12T06:50:14,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/17cf292a5c2a483eb8a9c71b75018099 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/17cf292a5c2a483eb8a9c71b75018099 2024-11-12T06:50:14,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/17cf292a5c2a483eb8a9c71b75018099, entries=150, sequenceid=818, filesize=12.0 K 2024-11-12T06:50:14,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/afd93e3d231f451b93e17505e63266ba as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/afd93e3d231f451b93e17505e63266ba 2024-11-12T06:50:14,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/afd93e3d231f451b93e17505e63266ba, entries=150, sequenceid=818, filesize=12.0 K 2024-11-12T06:50:14,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~304.50 KB/311808, currentSize=73.80 KB/75570 for f8bcab44974ca57906f4d57bd4289b93 in 879ms, sequenceid=818, compaction requested=false 2024-11-12T06:50:14,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:14,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-12T06:50:14,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:14,670 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing f8bcab44974ca57906f4d57bd4289b93 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T06:50:14,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=A 2024-11-12T06:50:14,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:14,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=B 2024-11-12T06:50:14,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:14,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=C 2024-11-12T06:50:14,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:14,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127e68a59e37334a5d94f91d01665acc5f_f8bcab44974ca57906f4d57bd4289b93 is 50, key is test_row_0/A:col10/1731394213763/Put/seqid=0 2024-11-12T06:50:14,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742429_1605 (size=12454) 2024-11-12T06:50:14,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:14,686 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127e68a59e37334a5d94f91d01665acc5f_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127e68a59e37334a5d94f91d01665acc5f_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:14,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/cb047046065d43a29e1968ecca7f15dd, store: [table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:14,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/cb047046065d43a29e1968ecca7f15dd is 175, key is test_row_0/A:col10/1731394213763/Put/seqid=0 2024-11-12T06:50:14,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742430_1606 (size=31255) 2024-11-12T06:50:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:14,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:14,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 321 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394274969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:14,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 317 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394274969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:14,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55628 deadline: 1731394274977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:14,981 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9f0beee3d6b6,38777,1731394047622 for 'test_row_0' in f8bcab44974ca57906f4d57bd4289b93 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18300 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T06:50:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T06:50:15,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 322 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394275077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:15,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 318 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394275077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:15,093 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=835, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/cb047046065d43a29e1968ecca7f15dd 2024-11-12T06:50:15,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/7cd71585f9c9481db9ccde4c6c4e83ca is 50, key is test_row_0/B:col10/1731394213763/Put/seqid=0 2024-11-12T06:50:15,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742431_1607 (size=12301) 2024-11-12T06:50:15,101 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=835 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/7cd71585f9c9481db9ccde4c6c4e83ca 2024-11-12T06:50:15,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/db2770193502453b8a852fc67237a77a is 50, key is test_row_0/C:col10/1731394213763/Put/seqid=0 2024-11-12T06:50:15,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742432_1608 (size=12301) 2024-11-12T06:50:15,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 323 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394275287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:15,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 319 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394275287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:15,517 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=835 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/db2770193502453b8a852fc67237a77a 2024-11-12T06:50:15,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/cb047046065d43a29e1968ecca7f15dd as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/cb047046065d43a29e1968ecca7f15dd 2024-11-12T06:50:15,524 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/cb047046065d43a29e1968ecca7f15dd, entries=150, sequenceid=835, filesize=30.5 K 2024-11-12T06:50:15,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/7cd71585f9c9481db9ccde4c6c4e83ca as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/7cd71585f9c9481db9ccde4c6c4e83ca 2024-11-12T06:50:15,528 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/7cd71585f9c9481db9ccde4c6c4e83ca, entries=150, sequenceid=835, filesize=12.0 K 2024-11-12T06:50:15,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/db2770193502453b8a852fc67237a77a as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/db2770193502453b8a852fc67237a77a 2024-11-12T06:50:15,532 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/db2770193502453b8a852fc67237a77a, entries=150, sequenceid=835, filesize=12.0 K 2024-11-12T06:50:15,533 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for f8bcab44974ca57906f4d57bd4289b93 in 862ms, sequenceid=835, compaction requested=true 2024-11-12T06:50:15,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:15,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:15,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-12T06:50:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-12T06:50:15,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-12T06:50:15,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6440 sec 2024-11-12T06:50:15,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.6470 sec 2024-11-12T06:50:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:15,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8bcab44974ca57906f4d57bd4289b93 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T06:50:15,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=A 2024-11-12T06:50:15,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:15,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=B 2024-11-12T06:50:15,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:15,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=C 2024-11-12T06:50:15,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:15,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125bae24cbc97e4d368ace9a40154f9f07_f8bcab44974ca57906f4d57bd4289b93 is 50, key is test_row_0/A:col10/1731394214967/Put/seqid=0 2024-11-12T06:50:15,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742433_1609 (size=14994) 2024-11-12T06:50:15,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 329 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394275619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:15,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:15,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:15,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:15,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 325 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394275620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:15,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:15,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 330 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394275727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:15,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:15,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394275727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:15,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:15,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:15,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 327 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394275937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:15,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 331 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394275937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:15,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:15,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:15,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:15,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:15,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:15,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:16,011 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:16,014 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125bae24cbc97e4d368ace9a40154f9f07_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125bae24cbc97e4d368ace9a40154f9f07_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:16,015 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/b9decbf3a73c4d9daf0c0639394e0af1, store: [table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:16,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/b9decbf3a73c4d9daf0c0639394e0af1 is 175, key is test_row_0/A:col10/1731394214967/Put/seqid=0 2024-11-12T06:50:16,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742434_1610 (size=39949) 2024-11-12T06:50:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T06:50:16,037 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:50:16,037 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T06:50:16,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-12T06:50:16,039 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T06:50:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T06:50:16,039 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T06:50:16,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T06:50:16,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T06:50:16,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-12T06:50:16,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:16,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:16,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:16,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:16,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:16,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 332 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394276247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:16,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:16,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 328 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394276247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:16,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:16,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:16,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:16,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-12T06:50:16,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:16,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:16,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:16,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:16,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T06:50:16,428 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=855, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/b9decbf3a73c4d9daf0c0639394e0af1 2024-11-12T06:50:16,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/804bdb9e8f0a4308a91f45cca62cc65e is 50, key is test_row_0/B:col10/1731394214967/Put/seqid=0 2024-11-12T06:50:16,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742435_1611 (size=12301) 2024-11-12T06:50:16,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-12T06:50:16,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:16,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:16,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:16,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:16,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:16,634 INFO [Thread-2305 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2305. 2024-11-12T06:50:16,634 DEBUG [Thread-2305 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:50:16,634 DEBUG [Thread-2305 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:16,634 DEBUG [Thread-2305 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:16,634 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:50:16,634 INFO [Thread-2307 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2307. 2024-11-12T06:50:16,634 DEBUG [Thread-2307 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:50:16,634 DEBUG [Thread-2307 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:16,635 DEBUG [Thread-2307 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:16,635 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:50:16,636 INFO [Thread-2309 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2309. 2024-11-12T06:50:16,636 DEBUG [Thread-2309 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:50:16,636 DEBUG [Thread-2309 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:16,636 DEBUG [Thread-2309 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:16,636 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:50:16,638 INFO [Thread-2303 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2303. 2024-11-12T06:50:16,638 DEBUG [Thread-2303 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:50:16,638 DEBUG [Thread-2303 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:16,638 DEBUG [Thread-2303 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:16,638 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:50:16,638 INFO [Thread-2301 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2301. 2024-11-12T06:50:16,638 DEBUG [Thread-2301 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:50:16,638 DEBUG [Thread-2301 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:16,638 DEBUG [Thread-2301 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:16,638 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:50:16,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-12T06:50:16,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:16,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:16,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:16,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:16,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:16,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:16,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T06:50:16,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:16,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T06:50:16,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 333 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55674 deadline: 1731394276758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:16,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 329 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1731394276758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 , the old value is region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T06:50:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., hostname=9f0beee3d6b6,38777,1731394047622, seqNum=5 because the exception is null or not the one we care about 2024-11-12T06:50:16,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-12T06:50:16,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:16,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:16,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:16,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:16,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:16,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=855 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/804bdb9e8f0a4308a91f45cca62cc65e 2024-11-12T06:50:16,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/dc2d581954b14810b7ae76bab2495e95 is 50, key is test_row_0/C:col10/1731394214967/Put/seqid=0 2024-11-12T06:50:16,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742436_1612 (size=12301) 2024-11-12T06:50:16,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-12T06:50:16,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:16,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:16,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:16,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:16,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:17,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-12T06:50:17,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:17,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:17,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:17,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:17,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T06:50:17,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=855 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/dc2d581954b14810b7ae76bab2495e95 2024-11-12T06:50:17,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/b9decbf3a73c4d9daf0c0639394e0af1 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/b9decbf3a73c4d9daf0c0639394e0af1 2024-11-12T06:50:17,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/b9decbf3a73c4d9daf0c0639394e0af1, entries=200, sequenceid=855, filesize=39.0 K 2024-11-12T06:50:17,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/804bdb9e8f0a4308a91f45cca62cc65e as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/804bdb9e8f0a4308a91f45cca62cc65e 2024-11-12T06:50:17,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/804bdb9e8f0a4308a91f45cca62cc65e, entries=150, sequenceid=855, filesize=12.0 K 2024-11-12T06:50:17,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/dc2d581954b14810b7ae76bab2495e95 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/dc2d581954b14810b7ae76bab2495e95 2024-11-12T06:50:17,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-12T06:50:17,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:17,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:17,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:17,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/dc2d581954b14810b7ae76bab2495e95, entries=150, sequenceid=855, filesize=12.0 K 2024-11-12T06:50:17,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:17,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:17,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f8bcab44974ca57906f4d57bd4289b93 in 1658ms, sequenceid=855, compaction requested=true 2024-11-12T06:50:17,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:17,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8bcab44974ca57906f4d57bd4289b93:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T06:50:17,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:17,260 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:50:17,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8bcab44974ca57906f4d57bd4289b93:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T06:50:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T06:50:17,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:17,260 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:50:17,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8bcab44974ca57906f4d57bd4289b93:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T06:50:17,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:50:17,261 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:50:17,261 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:50:17,261 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1541): f8bcab44974ca57906f4d57bd4289b93/A is initiating minor compaction (all files) 2024-11-12T06:50:17,261 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): f8bcab44974ca57906f4d57bd4289b93/B is initiating minor compaction (all files) 2024-11-12T06:50:17,261 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8bcab44974ca57906f4d57bd4289b93/B in TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:17,261 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8bcab44974ca57906f4d57bd4289b93/A in TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:17,261 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/40965b6d31974cab88f05fedd4fb64e7, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/f13b275afb4141e6a65844224d113dfc, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/cb047046065d43a29e1968ecca7f15dd, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/b9decbf3a73c4d9daf0c0639394e0af1] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp, totalSize=139.8 K 2024-11-12T06:50:17,261 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/a7bc1e5cb158451a907f5012e0f66903, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/17cf292a5c2a483eb8a9c71b75018099, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/7cd71585f9c9481db9ccde4c6c4e83ca, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/804bdb9e8f0a4308a91f45cca62cc65e] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp, totalSize=48.8 K 2024-11-12T06:50:17,261 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:17,261 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. files: [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/40965b6d31974cab88f05fedd4fb64e7, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/f13b275afb4141e6a65844224d113dfc, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/cb047046065d43a29e1968ecca7f15dd, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/b9decbf3a73c4d9daf0c0639394e0af1] 2024-11-12T06:50:17,261 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting a7bc1e5cb158451a907f5012e0f66903, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1731394211664 2024-11-12T06:50:17,261 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40965b6d31974cab88f05fedd4fb64e7, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1731394211664 2024-11-12T06:50:17,261 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 17cf292a5c2a483eb8a9c71b75018099, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=818, earliestPutTs=1731394213712 2024-11-12T06:50:17,262 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting f13b275afb4141e6a65844224d113dfc, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=818, earliestPutTs=1731394213712 2024-11-12T06:50:17,262 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cd71585f9c9481db9ccde4c6c4e83ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=835, earliestPutTs=1731394213751 2024-11-12T06:50:17,262 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb047046065d43a29e1968ecca7f15dd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=835, earliestPutTs=1731394213751 2024-11-12T06:50:17,262 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 804bdb9e8f0a4308a91f45cca62cc65e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=855, earliestPutTs=1731394214945 2024-11-12T06:50:17,262 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9decbf3a73c4d9daf0c0639394e0af1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=855, earliestPutTs=1731394214945 2024-11-12T06:50:17,267 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:17,268 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8bcab44974ca57906f4d57bd4289b93#B#compaction#507 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:50:17,269 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/f3fc1892afef48f2a97f05a282bb4649 is 50, key is test_row_0/B:col10/1731394214967/Put/seqid=0 2024-11-12T06:50:17,269 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112b7835f2795374fe09ea2b968de366a11_f8bcab44974ca57906f4d57bd4289b93 store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:17,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742437_1613 (size=13153) 2024-11-12T06:50:17,274 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112b7835f2795374fe09ea2b968de366a11_f8bcab44974ca57906f4d57bd4289b93, store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:17,274 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b7835f2795374fe09ea2b968de366a11_f8bcab44974ca57906f4d57bd4289b93 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:17,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742438_1614 (size=4469) 2024-11-12T06:50:17,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38777 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-12T06:50:17,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:17,412 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing f8bcab44974ca57906f4d57bd4289b93 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T06:50:17,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=A 2024-11-12T06:50:17,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:17,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=B 2024-11-12T06:50:17,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:17,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=C 2024-11-12T06:50:17,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:17,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124cd7b8429a134310a4f38e120925b45f_f8bcab44974ca57906f4d57bd4289b93 is 50, key is test_row_0/A:col10/1731394215618/Put/seqid=0 2024-11-12T06:50:17,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742439_1615 (size=12454) 2024-11-12T06:50:17,675 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/f3fc1892afef48f2a97f05a282bb4649 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/f3fc1892afef48f2a97f05a282bb4649 2024-11-12T06:50:17,677 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8bcab44974ca57906f4d57bd4289b93#A#compaction#508 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:50:17,678 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/d8f8ae4c0b9543e59a371bc15ac1bbdd is 175, key is test_row_0/A:col10/1731394214967/Put/seqid=0 2024-11-12T06:50:17,679 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8bcab44974ca57906f4d57bd4289b93/B of f8bcab44974ca57906f4d57bd4289b93 into f3fc1892afef48f2a97f05a282bb4649(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:50:17,679 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:17,679 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., storeName=f8bcab44974ca57906f4d57bd4289b93/B, priority=12, startTime=1731394217260; duration=0sec 2024-11-12T06:50:17,679 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T06:50:17,679 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8bcab44974ca57906f4d57bd4289b93:B 2024-11-12T06:50:17,679 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T06:50:17,680 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T06:50:17,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742440_1616 (size=32107) 2024-11-12T06:50:17,680 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1541): f8bcab44974ca57906f4d57bd4289b93/C is initiating minor compaction (all files) 2024-11-12T06:50:17,680 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8bcab44974ca57906f4d57bd4289b93/C in TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:17,680 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/75b392d35b5745818212c058cd2597d5, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/afd93e3d231f451b93e17505e63266ba, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/db2770193502453b8a852fc67237a77a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/dc2d581954b14810b7ae76bab2495e95] into tmpdir=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp, totalSize=48.8 K 2024-11-12T06:50:17,681 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting 75b392d35b5745818212c058cd2597d5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1731394211664 2024-11-12T06:50:17,681 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting afd93e3d231f451b93e17505e63266ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=818, earliestPutTs=1731394213712 2024-11-12T06:50:17,681 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting db2770193502453b8a852fc67237a77a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=835, earliestPutTs=1731394213751 2024-11-12T06:50:17,681 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] compactions.Compactor(225): Compacting dc2d581954b14810b7ae76bab2495e95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=855, earliestPutTs=1731394214945 2024-11-12T06:50:17,687 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8bcab44974ca57906f4d57bd4289b93#C#compaction#510 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T06:50:17,687 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/2c5569ae8fc44857824e22da54221f28 is 50, key is test_row_0/C:col10/1731394214967/Put/seqid=0 2024-11-12T06:50:17,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742441_1617 (size=13153) 2024-11-12T06:50:17,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8855): Flush requested on f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:17,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. as already flushing 2024-11-12T06:50:17,769 INFO [Thread-2290 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2290. 2024-11-12T06:50:17,769 DEBUG [Thread-2290 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:50:17,769 DEBUG [Thread-2290 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:17,769 INFO [Thread-2298 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2298. 2024-11-12T06:50:17,769 DEBUG [Thread-2298 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:50:17,769 DEBUG [Thread-2290 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:17,769 DEBUG [Thread-2298 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:17,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:50:17,769 DEBUG [Thread-2298 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:17,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:50:17,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:17,823 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124cd7b8429a134310a4f38e120925b45f_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124cd7b8429a134310a4f38e120925b45f_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:17,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/34d8f825b1094901988c48af038a0904, store: [table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:17,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/34d8f825b1094901988c48af038a0904 is 175, key is test_row_0/A:col10/1731394215618/Put/seqid=0 2024-11-12T06:50:17,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742442_1618 (size=31255) 2024-11-12T06:50:18,085 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/d8f8ae4c0b9543e59a371bc15ac1bbdd as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/d8f8ae4c0b9543e59a371bc15ac1bbdd 2024-11-12T06:50:18,088 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8bcab44974ca57906f4d57bd4289b93/A of f8bcab44974ca57906f4d57bd4289b93 into d8f8ae4c0b9543e59a371bc15ac1bbdd(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:50:18,088 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:18,088 INFO [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., storeName=f8bcab44974ca57906f4d57bd4289b93/A, priority=12, startTime=1731394217260; duration=0sec 2024-11-12T06:50:18,088 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:18,088 DEBUG [RS:0;9f0beee3d6b6:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8bcab44974ca57906f4d57bd4289b93:A 2024-11-12T06:50:18,093 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/2c5569ae8fc44857824e22da54221f28 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/2c5569ae8fc44857824e22da54221f28 2024-11-12T06:50:18,097 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8bcab44974ca57906f4d57bd4289b93/C of f8bcab44974ca57906f4d57bd4289b93 into 2c5569ae8fc44857824e22da54221f28(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T06:50:18,097 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:18,097 INFO [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., storeName=f8bcab44974ca57906f4d57bd4289b93/C, priority=12, startTime=1731394217260; duration=0sec 2024-11-12T06:50:18,097 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T06:50:18,097 DEBUG [RS:0;9f0beee3d6b6:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8bcab44974ca57906f4d57bd4289b93:C 2024-11-12T06:50:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T06:50:18,227 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=867, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/34d8f825b1094901988c48af038a0904 2024-11-12T06:50:18,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/77a5b38e3f0e440b8e4c38195b079d34 is 50, key is test_row_0/B:col10/1731394215618/Put/seqid=0 2024-11-12T06:50:18,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742443_1619 (size=12301) 2024-11-12T06:50:18,338 INFO [Thread-2296 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2296. 2024-11-12T06:50:18,338 DEBUG [Thread-2296 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:50:18,338 DEBUG [Thread-2296 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:18,338 DEBUG [Thread-2296 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:18,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:50:18,638 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=867 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/77a5b38e3f0e440b8e4c38195b079d34 2024-11-12T06:50:18,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/5dfec486867a4529aacded0fa4a0a9ad is 50, key is test_row_0/C:col10/1731394215618/Put/seqid=0 2024-11-12T06:50:18,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742444_1620 (size=12301) 2024-11-12T06:50:19,047 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=867 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/5dfec486867a4529aacded0fa4a0a9ad 2024-11-12T06:50:19,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/34d8f825b1094901988c48af038a0904 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/34d8f825b1094901988c48af038a0904 2024-11-12T06:50:19,053 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/34d8f825b1094901988c48af038a0904, entries=150, sequenceid=867, filesize=30.5 K 2024-11-12T06:50:19,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/77a5b38e3f0e440b8e4c38195b079d34 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/77a5b38e3f0e440b8e4c38195b079d34 2024-11-12T06:50:19,056 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/77a5b38e3f0e440b8e4c38195b079d34, entries=150, sequenceid=867, filesize=12.0 K 2024-11-12T06:50:19,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/5dfec486867a4529aacded0fa4a0a9ad as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/5dfec486867a4529aacded0fa4a0a9ad 2024-11-12T06:50:19,060 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/5dfec486867a4529aacded0fa4a0a9ad, entries=150, sequenceid=867, filesize=12.0 K 2024-11-12T06:50:19,061 INFO [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=20.13 KB/20610 for f8bcab44974ca57906f4d57bd4289b93 in 1650ms, sequenceid=867, compaction requested=false 2024-11-12T06:50:19,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for f8bcab44974ca57906f4d57bd4289b93: 2024-11-12T06:50:19,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:19,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f0beee3d6b6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-12T06:50:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-12T06:50:19,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-12T06:50:19,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0230 sec 2024-11-12T06:50:19,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 3.0270 sec 2024-11-12T06:50:20,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T06:50:20,197 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:50:20,998 INFO [Thread-2294 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2294. 2024-11-12T06:50:20,998 DEBUG [Thread-2294 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:50:20,998 DEBUG [Thread-2294 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:20,998 DEBUG [Thread-2294 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:20,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:50:25,021 INFO [Thread-2292 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2292. 2024-11-12T06:50:25,021 DEBUG [Thread-2292 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T06:50:25,021 DEBUG [Thread-2292 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:25,022 DEBUG [Thread-2292 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:25,022 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:50:25,022 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T06:50:25,022 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 279 2024-11-12T06:50:25,022 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-12T06:50:25,022 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-12T06:50:25,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 136 2024-11-12T06:50:25,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 280 2024-11-12T06:50:25,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T06:50:25,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2671 2024-11-12T06:50:25,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2535 2024-11-12T06:50:25,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2545 2024-11-12T06:50:25,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2668 2024-11-12T06:50:25,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2516 2024-11-12T06:50:25,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T06:50:25,023 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T06:50:25,023 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T06:50:25,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:25,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:25,024 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:50:25,025 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T06:50:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T06:50:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-12T06:50:25,030 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394225029"}]},"ts":"1731394225029"} 2024-11-12T06:50:25,032 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T06:50:25,032 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T06:50:25,033 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T06:50:25,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8bcab44974ca57906f4d57bd4289b93, UNASSIGN}] 2024-11-12T06:50:25,036 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8bcab44974ca57906f4d57bd4289b93, UNASSIGN 2024-11-12T06:50:25,036 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=f8bcab44974ca57906f4d57bd4289b93, regionState=CLOSING, regionLocation=9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:25,038 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8bcab44974ca57906f4d57bd4289b93, UNASSIGN because future has completed 2024-11-12T06:50:25,038 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T06:50:25,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622}] 2024-11-12T06:50:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-12T06:50:25,191 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:25,191 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T06:50:25,191 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing f8bcab44974ca57906f4d57bd4289b93, disabling compactions & flushes 2024-11-12T06:50:25,191 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:25,191 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:25,191 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. after waiting 0 ms 2024-11-12T06:50:25,191 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:25,191 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing f8bcab44974ca57906f4d57bd4289b93 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-12T06:50:25,191 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=A 2024-11-12T06:50:25,191 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:25,191 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=B 2024-11-12T06:50:25,191 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:25,191 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8bcab44974ca57906f4d57bd4289b93, store=C 2024-11-12T06:50:25,191 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T06:50:25,195 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125849718086444b92b3047d3a517fb4e7_f8bcab44974ca57906f4d57bd4289b93 is 50, key is test_row_0/A:col10/1731394225018/Put/seqid=0 2024-11-12T06:50:25,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742445_1621 (size=9914) 2024-11-12T06:50:25,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-12T06:50:25,599 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T06:50:25,603 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125849718086444b92b3047d3a517fb4e7_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125849718086444b92b3047d3a517fb4e7_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:25,604 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/2054ed93cb1b47a2acff3be8765cf8a1, store: [table=TestAcidGuarantees family=A region=f8bcab44974ca57906f4d57bd4289b93] 2024-11-12T06:50:25,605 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/2054ed93cb1b47a2acff3be8765cf8a1 is 175, key is test_row_0/A:col10/1731394225018/Put/seqid=0 2024-11-12T06:50:25,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742446_1622 (size=22561) 2024-11-12T06:50:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-12T06:50:25,970 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T06:50:26,010 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=878, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/2054ed93cb1b47a2acff3be8765cf8a1 2024-11-12T06:50:26,023 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/6d3e8b916d52465e879470cd3fd61fa3 is 50, key is test_row_0/B:col10/1731394225018/Put/seqid=0 2024-11-12T06:50:26,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742447_1623 (size=9857) 2024-11-12T06:50:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-12T06:50:26,429 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=878 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/6d3e8b916d52465e879470cd3fd61fa3 2024-11-12T06:50:26,434 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/6c0d22096f1c4667b588d7ae163b82b2 is 50, key is test_row_0/C:col10/1731394225018/Put/seqid=0 2024-11-12T06:50:26,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742448_1624 (size=9857) 2024-11-12T06:50:26,837 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=878 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/6c0d22096f1c4667b588d7ae163b82b2 2024-11-12T06:50:26,841 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/A/2054ed93cb1b47a2acff3be8765cf8a1 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/2054ed93cb1b47a2acff3be8765cf8a1 2024-11-12T06:50:26,844 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/2054ed93cb1b47a2acff3be8765cf8a1, entries=100, sequenceid=878, filesize=22.0 K 2024-11-12T06:50:26,844 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/B/6d3e8b916d52465e879470cd3fd61fa3 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/6d3e8b916d52465e879470cd3fd61fa3 2024-11-12T06:50:26,847 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/6d3e8b916d52465e879470cd3fd61fa3, entries=100, sequenceid=878, filesize=9.6 K 2024-11-12T06:50:26,847 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/.tmp/C/6c0d22096f1c4667b588d7ae163b82b2 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/6c0d22096f1c4667b588d7ae163b82b2 2024-11-12T06:50:26,849 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/6c0d22096f1c4667b588d7ae163b82b2, entries=100, sequenceid=878, filesize=9.6 K 2024-11-12T06:50:26,850 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f8bcab44974ca57906f4d57bd4289b93 in 1659ms, sequenceid=878, compaction requested=true 2024-11-12T06:50:26,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/27e766350b6b476686bf0d0c00509ad7, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/ecf247e30d634231a659dbc76031f400, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/ac3cb6da82a44fb995e034566ec4dbfc, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/94560b9f89654d2baa74df72fca93d1b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/63cbad3a0c3a4fdbbfd4bb72db3e34eb, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/e6a65b373dbc4caf887d17b14f6be391, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/efc1a96a8eac4a298139c2a31290d9df, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/17a6d5f2326f40d194eb2c98ac76ef83, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/29ba59bb13cb4b209cad124846308daf, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/f9a7728de5b2438b9b6dc2dc9288d5b5, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/2f3845f84dc4444796130684a3900109, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/a1a5dc9c667b41e392a33228a1b0258e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/899b0a43857b4d6b8dbf9a746c89e190, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/56416897c9a64a5584e9a9d47383ae7d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/d9346bffc4e24bb7953697902f56f07e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/641fd3beed9342df94389f730f8d5968, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/49fda5a6a24a4425a7d8b185fbd16644, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/634c2bf4aac64ddc9f866916d286c3ff, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/71829f68a2b24197a3a3f30ce6eebd9a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/395d6ad98f1f46a8a62ed9c7fbce8961, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/40965b6d31974cab88f05fedd4fb64e7, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/f771315f2da34c2fa85db1347f8a2a3a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/f13b275afb4141e6a65844224d113dfc, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/cb047046065d43a29e1968ecca7f15dd, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/b9decbf3a73c4d9daf0c0639394e0af1] to archive 2024-11-12T06:50:26,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T06:50:26,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/27e766350b6b476686bf0d0c00509ad7 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/27e766350b6b476686bf0d0c00509ad7 2024-11-12T06:50:26,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/ecf247e30d634231a659dbc76031f400 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/ecf247e30d634231a659dbc76031f400 2024-11-12T06:50:26,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/ac3cb6da82a44fb995e034566ec4dbfc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/ac3cb6da82a44fb995e034566ec4dbfc 2024-11-12T06:50:26,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/94560b9f89654d2baa74df72fca93d1b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/94560b9f89654d2baa74df72fca93d1b 2024-11-12T06:50:26,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/63cbad3a0c3a4fdbbfd4bb72db3e34eb to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/63cbad3a0c3a4fdbbfd4bb72db3e34eb 2024-11-12T06:50:26,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/e6a65b373dbc4caf887d17b14f6be391 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/e6a65b373dbc4caf887d17b14f6be391 2024-11-12T06:50:26,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/efc1a96a8eac4a298139c2a31290d9df to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/efc1a96a8eac4a298139c2a31290d9df 2024-11-12T06:50:26,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/17a6d5f2326f40d194eb2c98ac76ef83 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/17a6d5f2326f40d194eb2c98ac76ef83 2024-11-12T06:50:26,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/29ba59bb13cb4b209cad124846308daf to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/29ba59bb13cb4b209cad124846308daf 2024-11-12T06:50:26,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/f9a7728de5b2438b9b6dc2dc9288d5b5 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/f9a7728de5b2438b9b6dc2dc9288d5b5 2024-11-12T06:50:26,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/2f3845f84dc4444796130684a3900109 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/2f3845f84dc4444796130684a3900109 2024-11-12T06:50:26,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/a1a5dc9c667b41e392a33228a1b0258e to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/a1a5dc9c667b41e392a33228a1b0258e 2024-11-12T06:50:26,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/899b0a43857b4d6b8dbf9a746c89e190 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/899b0a43857b4d6b8dbf9a746c89e190 2024-11-12T06:50:26,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/56416897c9a64a5584e9a9d47383ae7d to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/56416897c9a64a5584e9a9d47383ae7d 2024-11-12T06:50:26,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/d9346bffc4e24bb7953697902f56f07e to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/d9346bffc4e24bb7953697902f56f07e 2024-11-12T06:50:26,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/641fd3beed9342df94389f730f8d5968 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/641fd3beed9342df94389f730f8d5968 2024-11-12T06:50:26,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/49fda5a6a24a4425a7d8b185fbd16644 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/49fda5a6a24a4425a7d8b185fbd16644 2024-11-12T06:50:26,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/634c2bf4aac64ddc9f866916d286c3ff to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/634c2bf4aac64ddc9f866916d286c3ff 2024-11-12T06:50:26,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/71829f68a2b24197a3a3f30ce6eebd9a to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/71829f68a2b24197a3a3f30ce6eebd9a 2024-11-12T06:50:26,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/395d6ad98f1f46a8a62ed9c7fbce8961 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/395d6ad98f1f46a8a62ed9c7fbce8961 2024-11-12T06:50:26,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/40965b6d31974cab88f05fedd4fb64e7 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/40965b6d31974cab88f05fedd4fb64e7 2024-11-12T06:50:26,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/f771315f2da34c2fa85db1347f8a2a3a to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/f771315f2da34c2fa85db1347f8a2a3a 2024-11-12T06:50:26,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/f13b275afb4141e6a65844224d113dfc to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/f13b275afb4141e6a65844224d113dfc 2024-11-12T06:50:26,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/cb047046065d43a29e1968ecca7f15dd to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/cb047046065d43a29e1968ecca7f15dd 2024-11-12T06:50:26,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/b9decbf3a73c4d9daf0c0639394e0af1 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/b9decbf3a73c4d9daf0c0639394e0af1 2024-11-12T06:50:26,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/eeae36d8b8004cc5b982d313da241cd0, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/b67ed65c42f9434084cfea89e1018be3, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/679ea1a922cd457fb7904fb3b8bf1b29, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/165fd00313904924a9c8c00adcecc3d8, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/2ffbe5d268554fadb607b91e99d3a23a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/f4079bd11aa14139adc725a19eaf4dab, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/57bc0625deda4ff4ad72e6fdbc78cb6b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/8bf8b4d30d3c44ce9903d891a373f38e, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/0b6eaccb80974eb0a99fbeb4b9ff64d9, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/757aff8019894c89aee6b9cbbe6066f4, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/253c35951d3b4bd8a9f0efa3cab40b15, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/1470e421fb304d6f81d429814ecbda56, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/03533005ff1041c9bb75a507509686e8, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/66917f8269e94639a8042ebbf5d24602, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/3433444266e4457684a11e7f726601f4, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/96a25c1be3b94acc9ff5a277d1cea5c8, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/24a490e1bb394a4c9cf2e77a1f671068, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/856d98a274f04cc087f9d3bfda54316a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/23c92fe9e693457a9cfd8fdfe9ae42f1, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/cfb8bc3f327f47dda9c0870de5b37a1d, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/a7bc1e5cb158451a907f5012e0f66903, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/d39db72369ec4ad3a7fafec5b787f29f, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/17cf292a5c2a483eb8a9c71b75018099, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/7cd71585f9c9481db9ccde4c6c4e83ca, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/804bdb9e8f0a4308a91f45cca62cc65e] to archive 2024-11-12T06:50:26,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T06:50:26,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/eeae36d8b8004cc5b982d313da241cd0 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/eeae36d8b8004cc5b982d313da241cd0 2024-11-12T06:50:26,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/b67ed65c42f9434084cfea89e1018be3 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/b67ed65c42f9434084cfea89e1018be3 2024-11-12T06:50:26,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/679ea1a922cd457fb7904fb3b8bf1b29 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/679ea1a922cd457fb7904fb3b8bf1b29 2024-11-12T06:50:26,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/165fd00313904924a9c8c00adcecc3d8 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/165fd00313904924a9c8c00adcecc3d8 2024-11-12T06:50:26,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/2ffbe5d268554fadb607b91e99d3a23a to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/2ffbe5d268554fadb607b91e99d3a23a 2024-11-12T06:50:26,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/f4079bd11aa14139adc725a19eaf4dab to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/f4079bd11aa14139adc725a19eaf4dab 2024-11-12T06:50:26,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/57bc0625deda4ff4ad72e6fdbc78cb6b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/57bc0625deda4ff4ad72e6fdbc78cb6b 2024-11-12T06:50:26,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/8bf8b4d30d3c44ce9903d891a373f38e to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/8bf8b4d30d3c44ce9903d891a373f38e 2024-11-12T06:50:26,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/0b6eaccb80974eb0a99fbeb4b9ff64d9 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/0b6eaccb80974eb0a99fbeb4b9ff64d9 2024-11-12T06:50:26,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/757aff8019894c89aee6b9cbbe6066f4 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/757aff8019894c89aee6b9cbbe6066f4 2024-11-12T06:50:26,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/253c35951d3b4bd8a9f0efa3cab40b15 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/253c35951d3b4bd8a9f0efa3cab40b15 2024-11-12T06:50:26,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/1470e421fb304d6f81d429814ecbda56 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/1470e421fb304d6f81d429814ecbda56 2024-11-12T06:50:26,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/03533005ff1041c9bb75a507509686e8 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/03533005ff1041c9bb75a507509686e8 2024-11-12T06:50:26,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/66917f8269e94639a8042ebbf5d24602 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/66917f8269e94639a8042ebbf5d24602 2024-11-12T06:50:26,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/3433444266e4457684a11e7f726601f4 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/3433444266e4457684a11e7f726601f4 2024-11-12T06:50:26,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/96a25c1be3b94acc9ff5a277d1cea5c8 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/96a25c1be3b94acc9ff5a277d1cea5c8 2024-11-12T06:50:26,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/24a490e1bb394a4c9cf2e77a1f671068 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/24a490e1bb394a4c9cf2e77a1f671068 2024-11-12T06:50:26,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/856d98a274f04cc087f9d3bfda54316a to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/856d98a274f04cc087f9d3bfda54316a 2024-11-12T06:50:26,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/23c92fe9e693457a9cfd8fdfe9ae42f1 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/23c92fe9e693457a9cfd8fdfe9ae42f1 2024-11-12T06:50:26,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/cfb8bc3f327f47dda9c0870de5b37a1d to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/cfb8bc3f327f47dda9c0870de5b37a1d 2024-11-12T06:50:26,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/a7bc1e5cb158451a907f5012e0f66903 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/a7bc1e5cb158451a907f5012e0f66903 2024-11-12T06:50:26,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/d39db72369ec4ad3a7fafec5b787f29f to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/d39db72369ec4ad3a7fafec5b787f29f 2024-11-12T06:50:26,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/17cf292a5c2a483eb8a9c71b75018099 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/17cf292a5c2a483eb8a9c71b75018099 2024-11-12T06:50:26,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/7cd71585f9c9481db9ccde4c6c4e83ca to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/7cd71585f9c9481db9ccde4c6c4e83ca 2024-11-12T06:50:26,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/804bdb9e8f0a4308a91f45cca62cc65e to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/804bdb9e8f0a4308a91f45cca62cc65e 2024-11-12T06:50:26,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/9168c3718c5244ca873721b493286347, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/f12ff1de4deb4e988b0a633965dec541, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/11c0797fbd6c49b9b115980903832157, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/e93a1f87da0e4ba1aed06eb3a93d2335, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/c56f21fb1c114803bd79f4aa58484210, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/cfaf3879cf5d4a9e909c4445d6e0e2cf, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/c8a690584e8848ce94f696e7cbf92d70, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/b2ddde96b1c24a4aac12478bf2506568, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/41a9c83c3e0248d580d9d07868660225, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/92afd40a2a274471a8f084648f4cc716, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/2e8eae894e3c43a882c1f738cb4f95d8, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/8a2a3ca4ab3b44d788faaf2045a21fdf, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/63af74117c5149b5ab266ef288b21800, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/96b68cc7f0d54dc684cf1a0aae989b33, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/a4f008844d4f4f3ea9e0368f33265d58, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/ac5088ec29a8498f8ca87c3886359d04, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/d9c25187dfbb42d0bde24e4314009076, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/bc7e5d35a6634b74829b1e6833008ea2, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/78f17e172a56426b9f5945473228c678, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/8cf6e9c3a4204a19829d30f8eec8929b, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/75b392d35b5745818212c058cd2597d5, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/0c2b09ecf1504589a799640b12afa1f6, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/afd93e3d231f451b93e17505e63266ba, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/db2770193502453b8a852fc67237a77a, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/dc2d581954b14810b7ae76bab2495e95] to archive 2024-11-12T06:50:26,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T06:50:26,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/9168c3718c5244ca873721b493286347 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/9168c3718c5244ca873721b493286347 2024-11-12T06:50:26,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/f12ff1de4deb4e988b0a633965dec541 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/f12ff1de4deb4e988b0a633965dec541 2024-11-12T06:50:26,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/11c0797fbd6c49b9b115980903832157 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/11c0797fbd6c49b9b115980903832157 2024-11-12T06:50:26,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/e93a1f87da0e4ba1aed06eb3a93d2335 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/e93a1f87da0e4ba1aed06eb3a93d2335 2024-11-12T06:50:26,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/c56f21fb1c114803bd79f4aa58484210 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/c56f21fb1c114803bd79f4aa58484210 2024-11-12T06:50:26,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/cfaf3879cf5d4a9e909c4445d6e0e2cf to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/cfaf3879cf5d4a9e909c4445d6e0e2cf 2024-11-12T06:50:26,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/c8a690584e8848ce94f696e7cbf92d70 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/c8a690584e8848ce94f696e7cbf92d70 2024-11-12T06:50:26,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/b2ddde96b1c24a4aac12478bf2506568 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/b2ddde96b1c24a4aac12478bf2506568 2024-11-12T06:50:26,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/41a9c83c3e0248d580d9d07868660225 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/41a9c83c3e0248d580d9d07868660225 2024-11-12T06:50:26,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/92afd40a2a274471a8f084648f4cc716 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/92afd40a2a274471a8f084648f4cc716 2024-11-12T06:50:26,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/2e8eae894e3c43a882c1f738cb4f95d8 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/2e8eae894e3c43a882c1f738cb4f95d8 2024-11-12T06:50:26,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/8a2a3ca4ab3b44d788faaf2045a21fdf to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/8a2a3ca4ab3b44d788faaf2045a21fdf 2024-11-12T06:50:26,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/63af74117c5149b5ab266ef288b21800 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/63af74117c5149b5ab266ef288b21800 2024-11-12T06:50:26,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/96b68cc7f0d54dc684cf1a0aae989b33 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/96b68cc7f0d54dc684cf1a0aae989b33 2024-11-12T06:50:26,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/a4f008844d4f4f3ea9e0368f33265d58 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/a4f008844d4f4f3ea9e0368f33265d58 2024-11-12T06:50:26,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/ac5088ec29a8498f8ca87c3886359d04 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/ac5088ec29a8498f8ca87c3886359d04 2024-11-12T06:50:26,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/d9c25187dfbb42d0bde24e4314009076 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/d9c25187dfbb42d0bde24e4314009076 2024-11-12T06:50:26,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/bc7e5d35a6634b74829b1e6833008ea2 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/bc7e5d35a6634b74829b1e6833008ea2 2024-11-12T06:50:26,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/78f17e172a56426b9f5945473228c678 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/78f17e172a56426b9f5945473228c678 2024-11-12T06:50:26,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/8cf6e9c3a4204a19829d30f8eec8929b to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/8cf6e9c3a4204a19829d30f8eec8929b 2024-11-12T06:50:26,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/75b392d35b5745818212c058cd2597d5 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/75b392d35b5745818212c058cd2597d5 2024-11-12T06:50:26,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/0c2b09ecf1504589a799640b12afa1f6 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/0c2b09ecf1504589a799640b12afa1f6 2024-11-12T06:50:26,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/afd93e3d231f451b93e17505e63266ba to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/afd93e3d231f451b93e17505e63266ba 2024-11-12T06:50:26,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/db2770193502453b8a852fc67237a77a to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/db2770193502453b8a852fc67237a77a 2024-11-12T06:50:26,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/dc2d581954b14810b7ae76bab2495e95 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/dc2d581954b14810b7ae76bab2495e95 2024-11-12T06:50:26,914 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/recovered.edits/881.seqid, newMaxSeqId=881, maxSeqId=4 2024-11-12T06:50:26,914 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. 2024-11-12T06:50:26,914 DEBUG [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for f8bcab44974ca57906f4d57bd4289b93: Waiting for close lock at 1731394225191Running coprocessor pre-close hooks at 1731394225191Disabling compacts and flushes for region at 1731394225191Disabling writes for close at 1731394225191Obtaining lock to block concurrent updates at 1731394225191Preparing flush snapshotting stores in f8bcab44974ca57906f4d57bd4289b93 at 1731394225191Finished memstore snapshotting TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731394225191Flushing stores of TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93. at 1731394225192 (+1 ms)Flushing f8bcab44974ca57906f4d57bd4289b93/A: creating writer at 1731394225192Flushing mob file f8bcab44974ca57906f4d57bd4289b93/A: appending metadata at 1731394225195 (+3 ms)Flushing mob file f8bcab44974ca57906f4d57bd4289b93/A: closing flushed file at 1731394225195Flushing f8bcab44974ca57906f4d57bd4289b93/A: appending metadata at 1731394225604 (+409 ms)Flushing f8bcab44974ca57906f4d57bd4289b93/A: closing flushed file at 1731394225604Flushing f8bcab44974ca57906f4d57bd4289b93/B: creating writer at 1731394226018 (+414 ms)Flushing f8bcab44974ca57906f4d57bd4289b93/B: appending metadata at 1731394226022 (+4 ms)Flushing f8bcab44974ca57906f4d57bd4289b93/B: closing flushed file at 1731394226022Flushing f8bcab44974ca57906f4d57bd4289b93/C: creating writer at 1731394226431 (+409 ms)Flushing f8bcab44974ca57906f4d57bd4289b93/C: appending metadata at 1731394226433 (+2 ms)Flushing f8bcab44974ca57906f4d57bd4289b93/C: closing flushed file at 1731394226433Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42cb8a7d: reopening flushed file at 1731394226840 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d8dfa9e: reopening flushed file at 1731394226844 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f3e3ac: reopening flushed file at 1731394226847 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f8bcab44974ca57906f4d57bd4289b93 in 1659ms, sequenceid=878, compaction requested=true at 1731394226850 (+3 ms)Writing region close event to WAL at 1731394226911 (+61 ms)Running coprocessor post-close hooks at 1731394226914 (+3 ms)Closed at 1731394226914 2024-11-12T06:50:26,916 INFO [RS_CLOSE_REGION-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:26,916 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=f8bcab44974ca57906f4d57bd4289b93, regionState=CLOSED 2024-11-12T06:50:26,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 because future has completed 2024-11-12T06:50:26,919 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-12T06:50:26,919 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure f8bcab44974ca57906f4d57bd4289b93, server=9f0beee3d6b6,38777,1731394047622 in 1.8800 sec 2024-11-12T06:50:26,920 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-11-12T06:50:26,920 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8bcab44974ca57906f4d57bd4289b93, UNASSIGN in 1.8850 sec 2024-11-12T06:50:26,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-12T06:50:26,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8880 sec 2024-11-12T06:50:26,923 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731394226922"}]},"ts":"1731394226922"} 2024-11-12T06:50:26,924 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T06:50:26,924 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T06:50:26,925 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8990 sec 2024-11-12T06:50:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-12T06:50:27,177 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:50:27,178 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T06:50:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:50:27,179 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:50:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-12T06:50:27,180 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:50:27,181 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,183 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A, FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B, FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C, FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/recovered.edits] 2024-11-12T06:50:27,185 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/2054ed93cb1b47a2acff3be8765cf8a1 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/2054ed93cb1b47a2acff3be8765cf8a1 2024-11-12T06:50:27,186 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/34d8f825b1094901988c48af038a0904 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/34d8f825b1094901988c48af038a0904 2024-11-12T06:50:27,187 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/d8f8ae4c0b9543e59a371bc15ac1bbdd to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/A/d8f8ae4c0b9543e59a371bc15ac1bbdd 2024-11-12T06:50:27,189 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/6d3e8b916d52465e879470cd3fd61fa3 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/6d3e8b916d52465e879470cd3fd61fa3 2024-11-12T06:50:27,189 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/77a5b38e3f0e440b8e4c38195b079d34 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/77a5b38e3f0e440b8e4c38195b079d34 2024-11-12T06:50:27,190 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/f3fc1892afef48f2a97f05a282bb4649 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/B/f3fc1892afef48f2a97f05a282bb4649 2024-11-12T06:50:27,191 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/2c5569ae8fc44857824e22da54221f28 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/2c5569ae8fc44857824e22da54221f28 2024-11-12T06:50:27,192 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/5dfec486867a4529aacded0fa4a0a9ad to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/5dfec486867a4529aacded0fa4a0a9ad 2024-11-12T06:50:27,193 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/6c0d22096f1c4667b588d7ae163b82b2 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/C/6c0d22096f1c4667b588d7ae163b82b2 2024-11-12T06:50:27,194 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/recovered.edits/881.seqid to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93/recovered.edits/881.seqid 2024-11-12T06:50:27,195 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/default/TestAcidGuarantees/f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,195 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T06:50:27,195 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T06:50:27,195 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-12T06:50:27,197 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120b0a8b3240a9468199fd1f1fcf11fe5a_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120b0a8b3240a9468199fd1f1fcf11fe5a_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,198 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121917dc8aaecf40a8bbb5abba5d219538_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121917dc8aaecf40a8bbb5abba5d219538_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,199 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111226a96be089fb44c1b5a9538f72a21a60_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111226a96be089fb44c1b5a9538f72a21a60_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,199 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122e0670529b5f4d74bf777ecfe61f4b37_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122e0670529b5f4d74bf777ecfe61f4b37_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,200 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123e1494e0fe8041ada8b8c3bb4ec21043_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123e1494e0fe8041ada8b8c3bb4ec21043_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,201 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123fa0b0e3389847e4844b7a3472d34d75_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123fa0b0e3389847e4844b7a3472d34d75_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,201 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111248a39b9191334c9cbe32a23d77dd49ec_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111248a39b9191334c9cbe32a23d77dd49ec_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,202 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124cd7b8429a134310a4f38e120925b45f_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124cd7b8429a134310a4f38e120925b45f_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,203 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125849718086444b92b3047d3a517fb4e7_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125849718086444b92b3047d3a517fb4e7_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,203 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125bae24cbc97e4d368ace9a40154f9f07_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125bae24cbc97e4d368ace9a40154f9f07_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,204 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111262527b28a6db4070899e8bd6554f49d9_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111262527b28a6db4070899e8bd6554f49d9_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,205 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112689cbbf8d1d04232965c867640d10f94_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112689cbbf8d1d04232965c867640d10f94_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,205 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111276bc8c412c0d435eb513008536e6c095_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111276bc8c412c0d435eb513008536e6c095_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,206 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127e68a59e37334a5d94f91d01665acc5f_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127e68a59e37334a5d94f91d01665acc5f_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,207 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111286b77b95ef694589b9a04f59e26b0435_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111286b77b95ef694589b9a04f59e26b0435_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,207 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111290db2ced9cab4989b20639d35d347e8d_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111290db2ced9cab4989b20639d35d347e8d_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,208 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a17b7e3e75de4f86a3b421aaf048f69a_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a17b7e3e75de4f86a3b421aaf048f69a_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,209 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112abc10da3d8e54d1485117e401082a4d7_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112abc10da3d8e54d1485117e401082a4d7_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,209 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ad5ceb27fdc54ef3a3781c10763f0f4d_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ad5ceb27fdc54ef3a3781c10763f0f4d_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,210 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112da9313a00ad24a92be28c8d912d470b2_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112da9313a00ad24a92be28c8d912d470b2_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,211 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112fae37d2f50f942c78b4dbbaa08cf0179_f8bcab44974ca57906f4d57bd4289b93 to hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112fae37d2f50f942c78b4dbbaa08cf0179_f8bcab44974ca57906f4d57bd4289b93 2024-11-12T06:50:27,211 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T06:50:27,213 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:50:27,214 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T06:50:27,216 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T06:50:27,217 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:50:27,217 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T06:50:27,217 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731394227217"}]},"ts":"9223372036854775807"} 2024-11-12T06:50:27,218 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T06:50:27,218 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f8bcab44974ca57906f4d57bd4289b93, NAME => 'TestAcidGuarantees,,1731394193524.f8bcab44974ca57906f4d57bd4289b93.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T06:50:27,218 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T06:50:27,218 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731394227218"}]},"ts":"9223372036854775807"} 2024-11-12T06:50:27,219 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T06:50:27,220 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T06:50:27,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-11-12T06:50:27,229 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T06:50:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-12T06:50:27,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T06:50:27,287 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T06:50:27,295 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 250), OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=323 (was 310) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6367 (was 6485) 2024-11-12T06:50:27,295 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T06:50:27,295 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T06:50:27,295 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T06:50:27,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:27,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:27,296 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T06:50:27,296 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-12T06:50:27,296 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=164229068, stopped=false 2024-11-12T06:50:27,296 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9f0beee3d6b6,43031,1731394046864 2024-11-12T06:50:27,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x1012f161b0d0001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T06:50:27,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T06:50:27,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x1012f161b0d0001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T06:50:27,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T06:50:27,353 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T06:50:27,354 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T06:50:27,354 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38777-0x1012f161b0d0001, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T06:50:27,354 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T06:50:27,354 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T06:50:27,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:27,354 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9f0beee3d6b6,38777,1731394047622' ***** 2024-11-12T06:50:27,354 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T06:50:27,355 INFO [RS:0;9f0beee3d6b6:38777 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T06:50:27,355 INFO [RS:0;9f0beee3d6b6:38777 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T06:50:27,355 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T06:50:27,355 INFO [RS:0;9f0beee3d6b6:38777 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T06:50:27,355 INFO [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(959): stopping server 9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:27,355 INFO [RS:0;9f0beee3d6b6:38777 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T06:50:27,355 INFO [RS:0;9f0beee3d6b6:38777 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9f0beee3d6b6:38777. 2024-11-12T06:50:27,355 DEBUG [RS:0;9f0beee3d6b6:38777 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T06:50:27,355 DEBUG [RS:0;9f0beee3d6b6:38777 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:27,355 INFO [RS:0;9f0beee3d6b6:38777 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T06:50:27,355 INFO [RS:0;9f0beee3d6b6:38777 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T06:50:27,355 INFO [RS:0;9f0beee3d6b6:38777 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T06:50:27,355 INFO [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T06:50:27,356 INFO [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-12T06:50:27,356 DEBUG [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-12T06:50:27,356 DEBUG [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T06:50:27,356 INFO [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T06:50:27,356 DEBUG [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T06:50:27,356 DEBUG [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T06:50:27,356 DEBUG [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T06:50:27,356 DEBUG [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T06:50:27,356 INFO [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-12T06:50:27,369 INFO [regionserver/9f0beee3d6b6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T06:50:27,377 DEBUG [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740/.tmp/info/9e1ce30b7db14f378423d37b9a0c8ac9 is 95, key is TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4./info:/1731394077659/DeleteFamily/seqid=0 2024-11-12T06:50:27,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742449_1625 (size=6514) 2024-11-12T06:50:27,556 DEBUG [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T06:50:27,660 INFO [regionserver/9f0beee3d6b6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-12T06:50:27,660 INFO [regionserver/9f0beee3d6b6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-12T06:50:27,757 DEBUG [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T06:50:27,781 INFO [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740/.tmp/info/9e1ce30b7db14f378423d37b9a0c8ac9 2024-11-12T06:50:27,798 DEBUG [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740/.tmp/ns/c3b435c145984eecad14d5ca199b1407 is 93, key is TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4./ns:/1731394077616/DeleteFamily/seqid=0 2024-11-12T06:50:27,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742450_1626 (size=5906) 2024-11-12T06:50:27,957 DEBUG [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T06:50:28,157 DEBUG [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T06:50:28,202 INFO [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740/.tmp/ns/c3b435c145984eecad14d5ca199b1407 2024-11-12T06:50:28,219 DEBUG [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740/.tmp/rep_barrier/718decde41894789b58330d953396380 is 102, key is TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4./rep_barrier:/1731394077616/DeleteFamily/seqid=0 2024-11-12T06:50:28,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742451_1627 (size=6025) 2024-11-12T06:50:28,357 DEBUG [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T06:50:28,557 DEBUG [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T06:50:28,623 INFO [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740/.tmp/rep_barrier/718decde41894789b58330d953396380 2024-11-12T06:50:28,638 DEBUG [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740/.tmp/table/35b63f039cc4400e842df89ee8b65365 is 96, key is TestAcidGuarantees,,1731394051859.c0610004ec3b741dc44aa538a4d26ae4./table:/1731394077616/DeleteFamily/seqid=0 2024-11-12T06:50:28,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742452_1628 (size=5936) 2024-11-12T06:50:28,758 DEBUG [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T06:50:28,958 DEBUG [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T06:50:29,041 INFO [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740/.tmp/table/35b63f039cc4400e842df89ee8b65365 2024-11-12T06:50:29,045 DEBUG [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740/.tmp/info/9e1ce30b7db14f378423d37b9a0c8ac9 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740/info/9e1ce30b7db14f378423d37b9a0c8ac9 2024-11-12T06:50:29,047 INFO [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740/info/9e1ce30b7db14f378423d37b9a0c8ac9, entries=12, sequenceid=90, filesize=6.4 K 2024-11-12T06:50:29,048 DEBUG [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740/.tmp/ns/c3b435c145984eecad14d5ca199b1407 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740/ns/c3b435c145984eecad14d5ca199b1407 2024-11-12T06:50:29,051 INFO [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740/ns/c3b435c145984eecad14d5ca199b1407, entries=8, sequenceid=90, filesize=5.8 K 2024-11-12T06:50:29,051 DEBUG [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740/.tmp/rep_barrier/718decde41894789b58330d953396380 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740/rep_barrier/718decde41894789b58330d953396380 2024-11-12T06:50:29,054 INFO [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740/rep_barrier/718decde41894789b58330d953396380, entries=6, sequenceid=90, filesize=5.9 K 2024-11-12T06:50:29,055 DEBUG [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740/.tmp/table/35b63f039cc4400e842df89ee8b65365 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740/table/35b63f039cc4400e842df89ee8b65365 2024-11-12T06:50:29,057 INFO [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740/table/35b63f039cc4400e842df89ee8b65365, entries=7, sequenceid=90, filesize=5.8 K 2024-11-12T06:50:29,057 INFO [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1701ms, sequenceid=90, compaction requested=false 2024-11-12T06:50:29,061 DEBUG [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-12T06:50:29,061 DEBUG [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T06:50:29,061 INFO [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T06:50:29,061 DEBUG [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731394227356Running coprocessor pre-close hooks at 1731394227356Disabling compacts and flushes for region at 1731394227356Disabling writes for close at 1731394227356Obtaining lock to block concurrent updates at 1731394227356Preparing flush snapshotting stores in 1588230740 at 1731394227356Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731394227357 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731394227357Flushing 1588230740/info: creating writer at 1731394227357Flushing 1588230740/info: appending metadata at 1731394227376 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731394227376Flushing 1588230740/ns: creating writer at 1731394227784 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731394227798 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731394227798Flushing 1588230740/rep_barrier: creating writer at 1731394228205 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731394228219 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731394228219Flushing 1588230740/table: creating writer at 1731394228625 (+406 ms)Flushing 1588230740/table: appending metadata at 1731394228638 (+13 ms)Flushing 1588230740/table: closing flushed file at 1731394228638Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68b8b24a: reopening flushed file at 1731394229044 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30596e71: reopening flushed file at 1731394229048 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@145decce: reopening flushed file at 1731394229051 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40f7647f: reopening flushed file at 1731394229054 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1701ms, sequenceid=90, compaction requested=false at 1731394229057 (+3 ms)Writing region close event to WAL at 1731394229059 (+2 ms)Running coprocessor post-close hooks at 1731394229061 (+2 ms)Closed at 1731394229061 2024-11-12T06:50:29,061 DEBUG [RS_CLOSE_META-regionserver/9f0beee3d6b6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T06:50:29,158 INFO [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(976): stopping server 9f0beee3d6b6,38777,1731394047622; all regions closed. 2024-11-12T06:50:29,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741834_1010 (size=25344) 2024-11-12T06:50:29,164 DEBUG [RS:0;9f0beee3d6b6:38777 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/oldWALs 2024-11-12T06:50:29,164 INFO [RS:0;9f0beee3d6b6:38777 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9f0beee3d6b6%2C38777%2C1731394047622.meta:.meta(num 1731394051427) 2024-11-12T06:50:29,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741833_1009 (size=27366299) 2024-11-12T06:50:29,167 DEBUG [RS:0;9f0beee3d6b6:38777 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/oldWALs 2024-11-12T06:50:29,167 INFO [RS:0;9f0beee3d6b6:38777 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9f0beee3d6b6%2C38777%2C1731394047622:(num 1731394050817) 2024-11-12T06:50:29,167 DEBUG [RS:0;9f0beee3d6b6:38777 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T06:50:29,167 INFO [RS:0;9f0beee3d6b6:38777 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T06:50:29,167 INFO [RS:0;9f0beee3d6b6:38777 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T06:50:29,168 INFO [RS:0;9f0beee3d6b6:38777 {}] hbase.ChoreService(370): Chore service for: regionserver/9f0beee3d6b6:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T06:50:29,168 INFO [RS:0;9f0beee3d6b6:38777 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T06:50:29,168 INFO [regionserver/9f0beee3d6b6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T06:50:29,169 INFO [RS:0;9f0beee3d6b6:38777 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38777 2024-11-12T06:50:29,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x1012f161b0d0001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9f0beee3d6b6,38777,1731394047622 2024-11-12T06:50:29,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T06:50:29,177 INFO [RS:0;9f0beee3d6b6:38777 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T06:50:29,178 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9f0beee3d6b6,38777,1731394047622] 2024-11-12T06:50:29,194 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9f0beee3d6b6,38777,1731394047622 already deleted, retry=false 2024-11-12T06:50:29,194 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9f0beee3d6b6,38777,1731394047622 expired; onlineServers=0 2024-11-12T06:50:29,194 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9f0beee3d6b6,43031,1731394046864' ***** 2024-11-12T06:50:29,195 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-12T06:50:29,195 INFO [M:0;9f0beee3d6b6:43031 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T06:50:29,195 INFO [M:0;9f0beee3d6b6:43031 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T06:50:29,195 DEBUG [M:0;9f0beee3d6b6:43031 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-12T06:50:29,195 DEBUG [M:0;9f0beee3d6b6:43031 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-12T06:50:29,195 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-12T06:50:29,195 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster-HFileCleaner.large.0-1731394050496 {}] cleaner.HFileCleaner(306): Exit Thread[master/9f0beee3d6b6:0:becomeActiveMaster-HFileCleaner.large.0-1731394050496,5,FailOnTimeoutGroup] 2024-11-12T06:50:29,195 DEBUG [master/9f0beee3d6b6:0:becomeActiveMaster-HFileCleaner.small.0-1731394050497 {}] cleaner.HFileCleaner(306): Exit Thread[master/9f0beee3d6b6:0:becomeActiveMaster-HFileCleaner.small.0-1731394050497,5,FailOnTimeoutGroup] 2024-11-12T06:50:29,195 INFO [M:0;9f0beee3d6b6:43031 {}] hbase.ChoreService(370): Chore service for: master/9f0beee3d6b6:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-12T06:50:29,195 INFO [M:0;9f0beee3d6b6:43031 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T06:50:29,195 DEBUG [M:0;9f0beee3d6b6:43031 {}] master.HMaster(1795): Stopping service threads 2024-11-12T06:50:29,195 INFO [M:0;9f0beee3d6b6:43031 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-12T06:50:29,195 INFO [M:0;9f0beee3d6b6:43031 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T06:50:29,195 ERROR [M:0;9f0beee3d6b6:43031 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-12T06:50:29,196 INFO [M:0;9f0beee3d6b6:43031 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T06:50:29,196 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-12T06:50:29,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-12T06:50:29,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T06:50:29,203 DEBUG [M:0;9f0beee3d6b6:43031 {}] zookeeper.ZKUtil(347): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-12T06:50:29,203 WARN [M:0;9f0beee3d6b6:43031 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-12T06:50:29,203 INFO [M:0;9f0beee3d6b6:43031 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/.lastflushedseqids 2024-11-12T06:50:29,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742453_1629 (size=72) 2024-11-12T06:50:29,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x1012f161b0d0001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T06:50:29,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x1012f161b0d0001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T06:50:29,287 INFO [RS:0;9f0beee3d6b6:38777 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T06:50:29,287 INFO [RS:0;9f0beee3d6b6:38777 {}] regionserver.HRegionServer(1031): Exiting; stopping=9f0beee3d6b6,38777,1731394047622; zookeeper connection closed. 2024-11-12T06:50:29,287 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@178ce89c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@178ce89c 2024-11-12T06:50:29,288 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-12T06:50:29,612 INFO [M:0;9f0beee3d6b6:43031 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T06:50:29,612 INFO [M:0;9f0beee3d6b6:43031 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-12T06:50:29,612 DEBUG [M:0;9f0beee3d6b6:43031 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T06:50:29,612 INFO [M:0;9f0beee3d6b6:43031 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T06:50:29,612 DEBUG [M:0;9f0beee3d6b6:43031 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T06:50:29,612 DEBUG [M:0;9f0beee3d6b6:43031 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T06:50:29,612 DEBUG [M:0;9f0beee3d6b6:43031 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T06:50:29,612 INFO [M:0;9f0beee3d6b6:43031 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=789.91 KB heapSize=968.61 KB 2024-11-12T06:50:29,627 DEBUG [M:0;9f0beee3d6b6:43031 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7b3af60020e24f5eb7647ec1480184ad is 82, key is hbase:meta,,1/info:regioninfo/1731394051513/Put/seqid=0 2024-11-12T06:50:29,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742454_1630 (size=5672) 2024-11-12T06:50:30,030 INFO [M:0;9f0beee3d6b6:43031 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2214 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7b3af60020e24f5eb7647ec1480184ad 2024-11-12T06:50:30,050 DEBUG [M:0;9f0beee3d6b6:43031 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/219e479f0c494c87bb84f4ebbe7871cd is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x8D/proc:d/1731394196589/Put/seqid=0 2024-11-12T06:50:30,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742455_1631 (size=45008) 2024-11-12T06:50:30,454 INFO [M:0;9f0beee3d6b6:43031 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=789.35 KB at sequenceid=2214 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/219e479f0c494c87bb84f4ebbe7871cd 2024-11-12T06:50:30,457 INFO [M:0;9f0beee3d6b6:43031 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 219e479f0c494c87bb84f4ebbe7871cd 2024-11-12T06:50:30,472 DEBUG [M:0;9f0beee3d6b6:43031 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a3106586b47d41329ae9271cbfd3b1b5 is 69, key is 9f0beee3d6b6,38777,1731394047622/rs:state/1731394050561/Put/seqid=0 2024-11-12T06:50:30,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073742456_1632 (size=5156) 2024-11-12T06:50:30,876 INFO [M:0;9f0beee3d6b6:43031 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2214 (bloomFilter=true), to=hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a3106586b47d41329ae9271cbfd3b1b5 2024-11-12T06:50:30,879 DEBUG [M:0;9f0beee3d6b6:43031 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7b3af60020e24f5eb7647ec1480184ad as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7b3af60020e24f5eb7647ec1480184ad 2024-11-12T06:50:30,882 INFO [M:0;9f0beee3d6b6:43031 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7b3af60020e24f5eb7647ec1480184ad, entries=8, sequenceid=2214, filesize=5.5 K 2024-11-12T06:50:30,882 DEBUG [M:0;9f0beee3d6b6:43031 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/219e479f0c494c87bb84f4ebbe7871cd as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/219e479f0c494c87bb84f4ebbe7871cd 2024-11-12T06:50:30,885 INFO [M:0;9f0beee3d6b6:43031 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 219e479f0c494c87bb84f4ebbe7871cd 2024-11-12T06:50:30,885 INFO [M:0;9f0beee3d6b6:43031 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/219e479f0c494c87bb84f4ebbe7871cd, entries=168, sequenceid=2214, filesize=44.0 K 2024-11-12T06:50:30,886 DEBUG [M:0;9f0beee3d6b6:43031 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a3106586b47d41329ae9271cbfd3b1b5 as hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a3106586b47d41329ae9271cbfd3b1b5 2024-11-12T06:50:30,888 INFO [M:0;9f0beee3d6b6:43031 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44429/user/jenkins/test-data/00e02fc4-66e9-346e-b104-2f726c29e76e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a3106586b47d41329ae9271cbfd3b1b5, entries=1, sequenceid=2214, filesize=5.0 K 2024-11-12T06:50:30,889 INFO [M:0;9f0beee3d6b6:43031 {}] regionserver.HRegion(3140): Finished flush of dataSize ~789.91 KB/808865, heapSize ~968.31 KB/991552, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1277ms, sequenceid=2214, compaction requested=false 2024-11-12T06:50:30,890 INFO [M:0;9f0beee3d6b6:43031 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T06:50:30,890 DEBUG [M:0;9f0beee3d6b6:43031 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731394229612Disabling compacts and flushes for region at 1731394229612Disabling writes for close at 1731394229612Obtaining lock to block concurrent updates at 1731394229612Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731394229612Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=808865, getHeapSize=991792, getOffHeapSize=0, getCellsCount=2407 at 1731394229612Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731394229613 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731394229613Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731394229626 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731394229626Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731394230033 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731394230049 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731394230049Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731394230457 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731394230472 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731394230472Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44e79ed9: reopening flushed file at 1731394230878 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35316be8: reopening flushed file at 1731394230882 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dea3dcb: reopening flushed file at 1731394230885 (+3 ms)Finished flush of dataSize ~789.91 KB/808865, heapSize ~968.31 KB/991552, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1277ms, sequenceid=2214, compaction requested=false at 1731394230889 (+4 ms)Writing region close event to WAL at 1731394230890 (+1 ms)Closed at 1731394230890 2024-11-12T06:50:30,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46017 is added to blk_1073741830_1006 (size=953966) 2024-11-12T06:50:30,893 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T06:50:30,893 INFO [M:0;9f0beee3d6b6:43031 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-12T06:50:30,893 INFO [M:0;9f0beee3d6b6:43031 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43031 2024-11-12T06:50:30,893 INFO [M:0;9f0beee3d6b6:43031 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T06:50:31,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T06:50:31,052 INFO [M:0;9f0beee3d6b6:43031 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T06:50:31,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43031-0x1012f161b0d0000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T06:50:31,057 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T06:50:31,059 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T06:50:31,059 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T06:50:31,059 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T06:50:31,059 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/hadoop.log.dir/,STOPPED} 2024-11-12T06:50:31,061 WARN [BP-229716150-172.17.0.2-1731394043510 heartbeating to localhost/127.0.0.1:44429 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T06:50:31,061 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T06:50:31,061 WARN [BP-229716150-172.17.0.2-1731394043510 heartbeating to localhost/127.0.0.1:44429 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-229716150-172.17.0.2-1731394043510 (Datanode Uuid 71d4d1fb-3c76-4c83-bad6-78ee8fa1cd38) service to localhost/127.0.0.1:44429 2024-11-12T06:50:31,061 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T06:50:31,065 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/cluster_a8cb0a6a-8309-0165-eb73-378e9ec04732/data/data1/current/BP-229716150-172.17.0.2-1731394043510 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T06:50:31,065 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/cluster_a8cb0a6a-8309-0165-eb73-378e9ec04732/data/data2/current/BP-229716150-172.17.0.2-1731394043510 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T06:50:31,066 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T06:50:31,073 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T06:50:31,073 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T06:50:31,073 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T06:50:31,074 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T06:50:31,074 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f413f13c-afe2-13c0-6b9f-ed13a127528d/hadoop.log.dir/,STOPPED} 2024-11-12T06:50:31,087 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-12T06:50:31,206 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down